diff --git a/.zuul.yaml b/.zuul.yaml
index c820c8e..d73be8f 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -29,10 +29,9 @@
       - playbooks/zuul-stream/.*
 
 - project:
-    name: openstack-infra/zuul
     check:
       jobs:
-        - build-openstack-sphinx-docs:
+        - build-sphinx-docs:
             irrelevant-files:
               - zuul/cmd/migrate.py
               - playbooks/zuul-migrate/.*
@@ -46,7 +45,7 @@
         - zuul-stream-functional
     gate:
       jobs:
-        - build-openstack-sphinx-docs:
+        - build-sphinx-docs:
             irrelevant-files:
               - zuul/cmd/migrate.py
               - playbooks/zuul-migrate/.*
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index d6b0984..ba14752 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -71,7 +71,7 @@
    [zookeeper]
    hosts=zk1.example.com,zk2.example.com,zk3.example.com
 
-   [webapp]
+   [web]
    status_url=https://zuul.example.com/status
 
    [scheduler]
@@ -234,22 +234,7 @@
 
       An openssl file containing the server private key in PEM format.
 
-.. attr:: webapp
-
-   .. attr:: listen_address
-      :default: all addresses
-
-      IP address or domain name on which to listen.
-
-   .. attr:: port
-      :default: 8001
-
-      Port on which the webapp is listening.
-
-   .. attr:: status_expiry
-      :default: 1
-
-      Zuul will cache the status.json file for this many seconds.
+.. attr:: web
 
    .. attr:: status_url
 
@@ -457,6 +442,11 @@
 
       Port to use for finger log streamer.
 
+   .. attr:: state_dir
+      :default: /var/lib/zuul
+
+      Path to directory in which Zuul should save its state.
+
    .. attr:: git_dir
       :default: /var/lib/zuul/git
 
@@ -575,6 +565,16 @@
       The executor will observe system load and determine whether
       to accept more jobs every 30 seconds.
 
+   .. attr:: min_avail_mem
+      :default: 5.0
+
+      This is the minimum percentage of system RAM available. The
+      executor will stop accepting more than 1 job at a time until
+      more memory is available. The available memory percentage is
+      calculated from the total available memory divided by the
+      total real memory multiplied by 100. Buffers and cache are
+      considered available in the calculation.
+
    .. attr:: hostname
       :default: hostname of the server
 
diff --git a/doc/source/admin/drivers/github.rst b/doc/source/admin/drivers/github.rst
index 4f46af6..83ac77f 100644
--- a/doc/source/admin/drivers/github.rst
+++ b/doc/source/admin/drivers/github.rst
@@ -317,10 +317,10 @@
       reporter should set as the commit status on github.
 
    .. TODO support role markup in :default: so we can xref
-      :attr:`webapp.status_url` below
+      :attr:`web.status_url` below
 
    .. attr:: status-url
-      :default: webapp.status_url or the empty string
+      :default: web.status_url or the empty string
 
       String value for a link url to set in the github
       status. Defaults to the zuul server status_url, or the empty
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index e6e6139..1c17c28 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -26,7 +26,7 @@
 
 These metrics are emitted by the Zuul :ref:`scheduler`:
 
-.. stat:: zuul.event.<driver>.event.<type>
+.. stat:: zuul.event.<driver>.<type>
    :type: counter
 
    Zuul will report counters for each type of event it receives from
@@ -131,21 +131,63 @@
    component of the key will be replaced with the hostname of the
    executor.
 
+   .. stat:: merger.<result>
+      :type: counter
+
+      Incremented to represent the status of a Zuul executor's merger
+      operations. ``<result>`` can be either ``SUCCESS`` or ``FAILURE``.
+      A failed merge operation which would be accounted for as a ``FAILURE``
+      is what ends up being returned by Zuul as a ``MERGER_FAILURE``.
+
    .. stat:: builds
       :type: counter
 
       Incremented each time the executor starts a build.
 
+   .. stat:: starting_builds
+      :type: gauge
+
+      The number of builds starting on this executor.  These are
+      builds which have not yet begun their first pre-playbook.
+
    .. stat:: running_builds
       :type: gauge
 
-      The number of builds currently running on this executor.
+      The number of builds currently running on this executor.  This
+      includes starting builds.
+
+  .. stat:: phase
+
+     Subtree detailing per-phase execution statistics:
+
+     .. stat:: <phase>
+
+        ``<phase>`` represents a phase in the execution of a job.
+        This can be an *internal* phase (such as ``setup`` or ``cleanup``) as
+        well as *job* phases such as ``pre``, ``run`` or ``post``.
+
+        .. stat:: <result>
+           :type: counter
+
+           A counter for each type of result.
+           These results do not, by themselves, determine the status of a build
+           but are indicators of the exit status provided by Ansible for the
+           execution of a particular phase.
+
+           Example of possible counters for each phase are: ``RESULT_NORMAL``,
+           ``RESULT_TIMED_OUT``, ``RESULT_UNREACHABLE``, ``RESULT_ABORTED``.
 
    .. stat:: load_average
       :type: gauge
 
       The one-minute load average of this executor, multiplied by 100.
 
+   .. stat:: pct_available_ram
+      :type: gauge
+
+      The available RAM (including buffers and cache) on this
+      executor, as a percentage multiplied by 100.
+
 .. stat:: zuul.nodepool
 
    Holds metrics related to Zuul requests from Nodepool.
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index 48e7ba8..5bcd2a2 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -25,7 +25,7 @@
 ------
 
 A tenant is a collection of projects which share a Zuul
-configuration.  An example tenant definition is:
+configuration. Some examples of tenant definitions are:
 
 .. code-block:: yaml
 
@@ -46,6 +46,27 @@
              - project2:
                  exclude-unprotected-branches: true
 
+.. code-block:: yaml
+
+   - tenant:
+       name: my-tenant
+       source:
+         gerrit:
+           config-projects:
+             - common-config
+           untrusted-projects:
+             - exclude:
+                 - job
+                 - semaphore
+                 - project
+                 - project-template
+                 - nodeset
+                 - secret
+               projects:
+                 - project1
+                 - project2:
+                     exclude-unprotected-branches: true
+
 .. attr:: tenant
 
    The following attributes are supported:
@@ -157,6 +178,24 @@
             processed. Defaults to the tenant wide setting of
             exclude-unprotected-branches.
 
+      .. attr:: <project-group>
+
+         The items in the list are dictionaries with the following
+         attributes. A **configuration items** definition is applied
+         to the list of projects.
+
+         .. attr:: include
+
+            A list of **configuration items** that should be loaded.
+
+         .. attr:: exclude
+
+            A list of **configuration items** that should not be loaded.
+
+         .. attr:: projects
+
+            A list of **project** items.
+
    .. attr:: max-nodes-per-job
       :default: 5
 
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 525cb38..597062e 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -692,6 +692,11 @@
       attribute to apply this behavior to a subset of a job's
       projects.
 
+      This value is also used to help select which variants of a job
+      to run.  If ``override-checkout`` is set, then Zuul will use
+      this value instead of the branch of the item being tested when
+      collecting jobs to run.
+
    .. attr:: timeout
 
       The time in seconds that the job should be allowed to run before
@@ -837,6 +842,12 @@
          :attr:`job.override-checkout` attribute to apply the same
          behavior to all projects in a job.
 
+         This value is also used to help select which variants of a
+         job to run.  If ``override-checkout`` is set, then Zuul will
+         use this value instead of the branch of the item being tested
+         when collecting any jobs to run which are defined in this
+         project.
+
    .. attr:: vars
 
       A dictionary of variables to supply to Ansible.  When inheriting
@@ -895,6 +906,12 @@
       branch of an item, then that job is not run for the item.
       Otherwise, all of the job variants which match that branch (and
       any other selection criteria) are used when freezing the job.
+      However, if :attr:`job.override-checkout` or
+      :attr:`job.required-projects.override-checkout` are set for a
+      project, Zuul will attempt to use the job variants which match
+      the values supplied in ``override-checkout`` for jobs defined in
+      those projects.  This can be used to run a job defined in one
+      project on another project without a matching branch.
 
       This example illustrates a job called *run-tests* which uses a
       nodeset based on the current release of an operating system to
@@ -1137,8 +1154,12 @@
 encrypted, however, data which are not sensitive may be provided
 unencrypted as well for convenience.
 
-A Secret may only be used by jobs defined within the same project.  To
-use a secret, a :ref:`job` must specify the secret in
+A Secret may only be used by jobs defined within the same project.
+Note that they can be used by any branch of that project, so if a
+project's branches have different access controls, consider whether
+all branches of that project are equally trusted before using secrets.
+
+To use a secret, a :ref:`job` must specify the secret in
 :attr:`job.secrets`.  Secrets are bound to the playbooks associated
 with the specific job definition where they were declared.  Additional
 pre or post playbooks which appear in child jobs will not have access
@@ -1175,6 +1196,12 @@
 `allowed-projects` job attribute can be used to restrict the projects
 which can invoke that job.
 
+Secrets, like most configuration items, are unique within a tenant,
+though a secret may be defined on multiple branches of the same
+project as long as the contents are the same.  This is to aid in
+branch maintenance, so that creating a new branch based on an existing
+branch will not immediately produce a configuration error.
+
 .. attr:: secret
 
    The following attributes must appear on a secret:
@@ -1203,6 +1230,12 @@
 groups of node types once and referring to them by name, job
 configuration may be simplified.
 
+Nodesets, like most configuration items, are unique within a tenant,
+though a nodeset may be defined on multiple branches of the same
+project as long as the contents are the same.  This is to aid in
+branch maintenance, so that creating a new branch based on an existing
+branch will not immediately produce a configuration error.
+
 .. code-block:: yaml
 
    - nodeset:
@@ -1285,9 +1318,19 @@
 represents the maximum number of jobs which use that semaphore at the
 same time.
 
+Semaphores, like most configuration items, are unique within a tenant,
+though a semaphore may be defined on multiple branches of the same
+project as long as the value is the same.  This is to aid in branch
+maintenance, so that creating a new branch based on an existing branch
+will not immediately produce a configuration error.
+
 Semaphores are never subject to dynamic reconfiguration.  If the value
 of a semaphore is changed, it will take effect only when the change
-where it is updated is merged.  An example follows:
+where it is updated is merged.  However, Zuul will attempt to validate
+the configuration of semaphores in proposed updates, even if they
+aren't used.
+
+An example usage of semaphores follows:
 
 .. code-block:: yaml
 
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 9ec4646..820e316 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -281,14 +281,6 @@
             msg: "Project {{ item.name }} is at {{ item.src_dir }}
           with_items: {{ zuul.projects.values() | list }}
 
-
-   .. var:: _projects
-      :type: dict
-
-      The same as ``projects`` but a dictionary indexed by the
-      ``name`` value of each entry.  ``projects`` will be converted to
-      this.
-
    .. var:: tenant
 
       The name of the current Zuul tenant.
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index 17092af..62b5086 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -39,10 +39,6 @@
 port=9000
 static_cache_expiry=0
 ;sql_connection_name=mydatabase
-
-[webapp]
-listen_address=0.0.0.0
-port=8001
 status_url=https://zuul.example.com/status
 
 [connection gerrit]
diff --git a/requirements.txt b/requirements.txt
index 39a2b02..7057c5a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@
 PyYAML>=3.1.0
 Paste
 WebOb>=1.2.3
-paramiko>=1.8.0,<2.0.0
+paramiko>=2.0.1
 GitPython>=2.1.8
 python-daemon>=2.0.4,<2.1.0
 extras
@@ -25,5 +25,6 @@
 cachecontrol
 pyjwt
 iso8601
-aiohttp
+aiohttp<3.0.0
 uvloop;python_version>='3.5'
+psutil
diff --git a/test-requirements.txt b/test-requirements.txt
index b444297..70f8e78 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,12 +1,9 @@
-pep8
-pyflakes
 flake8
 
 coverage>=3.6
 sphinx>=1.5.1,<1.6
 sphinxcontrib-blockdiag>=1.1.0
 fixtures>=0.3.14
-python-keystoneclient>=0.4.2
 python-subunit
 testrepository>=0.0.17
 testtools>=0.9.32
diff --git a/tests/base.py b/tests/base.py
index c449242..70889bb 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -57,7 +57,6 @@
 import zuul.driver.gerrit.gerritconnection as gerritconnection
 import zuul.driver.github.githubconnection as githubconnection
 import zuul.scheduler
-import zuul.webapp
 import zuul.executor.server
 import zuul.executor.client
 import zuul.lib.connections
@@ -66,9 +65,11 @@
 import zuul.merger.server
 import zuul.model
 import zuul.nodepool
+import zuul.rpcclient
 import zuul.zk
 import zuul.configloader
 from zuul.exceptions import MergeFailure
+from zuul.lib.config import get_default
 
 FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
                            'fixtures')
@@ -606,7 +607,7 @@
         return ret
 
     def getGitUrl(self, project):
-        return os.path.join(self.upstream_root, project.name)
+        return 'file://' + os.path.join(self.upstream_root, project.name)
 
 
 class GithubChangeReference(git.Reference):
@@ -939,7 +940,7 @@
 class FakeGithubConnection(githubconnection.GithubConnection):
     log = logging.getLogger("zuul.test.FakeGithubConnection")
 
-    def __init__(self, driver, connection_name, connection_config,
+    def __init__(self, driver, connection_name, connection_config, rpcclient,
                  changes_db=None, upstream_root=None):
         super(FakeGithubConnection, self).__init__(driver, connection_name,
                                                    connection_config)
@@ -952,12 +953,16 @@
         self.merge_not_allowed_count = 0
         self.reports = []
         self.github_client = tests.fakegithub.FakeGithub(changes_db)
+        self.rpcclient = rpcclient
 
     def getGithubClient(self,
                         project=None,
                         user_id=None):
         return self.github_client
 
+    def setZuulWebPort(self, port):
+        self.zuul_web_port = port
+
     def openFakePullRequest(self, project, branch, subject, files=[],
                             body=None):
         self.pr_number += 1
@@ -991,19 +996,25 @@
         }
         return (name, data)
 
-    def emitEvent(self, event):
+    def emitEvent(self, event, use_zuulweb=False):
         """Emulates sending the GitHub webhook event to the connection."""
-        port = self.webapp.server.socket.getsockname()[1]
         name, data = event
         payload = json.dumps(data).encode('utf8')
         secret = self.connection_config['webhook_token']
         signature = githubconnection._sign_request(payload, secret)
-        headers = {'X-Github-Event': name, 'X-Hub-Signature': signature}
-        req = urllib.request.Request(
-            'http://localhost:%s/connection/%s/payload'
-            % (port, self.connection_name),
-            data=payload, headers=headers)
-        return urllib.request.urlopen(req)
+        headers = {'x-github-event': name, 'x-hub-signature': signature}
+
+        if use_zuulweb:
+            req = urllib.request.Request(
+                'http://127.0.0.1:%s/connection/%s/payload'
+                % (self.zuul_web_port, self.connection_name),
+                data=payload, headers=headers)
+            return urllib.request.urlopen(req)
+        else:
+            job = self.rpcclient.submitJob(
+                'github:%s:payload' % self.connection_name,
+                {'headers': headers, 'body': data})
+            return json.loads(job.data[0])
 
     def addProject(self, project):
         # use the original method here and additionally register it in the
@@ -1629,6 +1640,10 @@
         nodeid = path.split("/")[-1]
         return nodeid
 
+    def removeNode(self, node):
+        path = self.NODE_ROOT + '/' + node["_oid"]
+        self.client.delete(path, recursive=True)
+
     def addFailRequest(self, request):
         self.fail_requests.add(request['_oid'])
 
@@ -1794,18 +1809,6 @@
         else:
             self._log_stream = sys.stdout
 
-        # NOTE(jeblair): this is temporary extra debugging to try to
-        # track down a possible leak.
-        orig_git_repo_init = git.Repo.__init__
-
-        def git_repo_init(myself, *args, **kw):
-            orig_git_repo_init(myself, *args, **kw)
-            self.log.debug("Created git repo 0x%x %s" %
-                           (id(myself), repr(myself)))
-
-        self.useFixture(fixtures.MonkeyPatch('git.Repo.__init__',
-                                             git_repo_init))
-
         handler = logging.StreamHandler(self._log_stream)
         formatter = logging.Formatter('%(asctime)s %(name)-32s '
                                       '%(levelname)-8s %(message)s')
@@ -1960,6 +1963,9 @@
         self.config.set(
             'executor', 'command_socket',
             os.path.join(self.test_root, 'executor.socket'))
+        self.config.set(
+            'merger', 'command_socket',
+            os.path.join(self.test_root, 'merger.socket'))
 
         self.statsd = FakeStatsd()
         if self.config.has_section('statsd'):
@@ -1983,6 +1989,13 @@
                 'gearman', 'ssl_key',
                 os.path.join(FIXTURE_DIR, 'gearman/client.key'))
 
+        self.rpcclient = zuul.rpcclient.RPCClient(
+            self.config.get('gearman', 'server'),
+            self.gearman_server.port,
+            get_default(self.config, 'gearman', 'ssl_key'),
+            get_default(self.config, 'gearman', 'ssl_cert'),
+            get_default(self.config, 'gearman', 'ssl_ca'))
+
         gerritsource.GerritSource.replication_timeout = 1.5
         gerritsource.GerritSource.replication_retry_interval = 0.5
         gerritconnection.GerritEventConnector.delay = 0.0
@@ -1990,9 +2003,6 @@
         self.sched = zuul.scheduler.Scheduler(self.config)
         self.sched._stats_interval = 1
 
-        self.webapp = zuul.webapp.WebApp(
-            self.sched, port=0, listen_address='127.0.0.1')
-
         self.event_queues = [
             self.sched.result_event_queue,
             self.sched.trigger_event_queue,
@@ -2000,7 +2010,7 @@
         ]
 
         self.configure_connections()
-        self.sched.registerConnections(self.connections, self.webapp)
+        self.sched.registerConnections(self.connections)
 
         self.executor_server = RecordingExecutorServer(
             self.config, self.connections,
@@ -2016,6 +2026,7 @@
             self.config, self.sched)
         self.merge_client = zuul.merger.client.MergeClient(
             self.config, self.sched)
+        self.merge_server = None
         self.nodepool = zuul.nodepool.Nodepool(self.sched)
         self.zk = zuul.zk.ZooKeeper()
         self.zk.connect(self.zk_config)
@@ -2031,7 +2042,6 @@
         self.sched.setZooKeeper(self.zk)
 
         self.sched.start()
-        self.webapp.start()
         self.executor_client.gearman.waitForServer()
         # Cleanups are run in reverse order
         self.addCleanup(self.assertCleanShutdown)
@@ -2065,6 +2075,7 @@
             server = config.get('server', 'github.com')
             db = self.github_changes_dbs.setdefault(server, {})
             con = FakeGithubConnection(driver, name, config,
+                                       self.rpcclient,
                                        changes_db=db,
                                        upstream_root=self.upstream_root)
             self.event_queues.append(con.event_queue)
@@ -2290,13 +2301,14 @@
         self.executor_server.release()
         self.executor_client.stop()
         self.merge_client.stop()
+        if self.merge_server:
+            self.merge_server.stop()
         self.executor_server.stop()
         self.sched.stop()
         self.sched.join()
         self.statsd.stop()
         self.statsd.join()
-        self.webapp.stop()
-        self.webapp.join()
+        self.rpcclient.shutdown()
         self.gearman_server.shutdown()
         self.fake_nodepool.stop()
         self.zk.disconnect()
@@ -2362,6 +2374,13 @@
         zuul.merger.merger.reset_repo_to_head(repo)
         repo.git.clean('-x', '-f', '-d')
 
+    def delete_branch(self, project, branch):
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo(path)
+        repo.head.reference = repo.heads['master']
+        zuul.merger.merger.reset_repo_to_head(repo)
+        repo.delete_head(repo.heads[branch], force=True)
+
     def create_commit(self, project):
         path = os.path.join(self.upstream_root, project)
         repo = git.Repo(path)
diff --git a/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml b/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml b/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml
new file mode 100644
index 0000000..3000df5
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml
@@ -0,0 +1,27 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    run: playbooks/base.yaml
+    parent: null
+
+- job:
+    name: restricted-job
+    allowed-projects:
+      - org/project1
+    
+- project:
+    name: common-config
+    check:
+      jobs: []
diff --git a/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..d3c98f3
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml
@@ -0,0 +1,10 @@
+- job:
+    name: test-project1
+    parent: restricted-job
+      
+- project:
+    name: org/project1
+    check:
+      jobs:
+        - test-project1
+        - restricted-job
diff --git a/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..bf0f07a
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+    name: test-project2
+    parent: restricted-job
+    allowed-projects:
+      - org/project2
+    
+- project:
+    name: org/project2
+    check:
+      jobs:
+        - test-project2
diff --git a/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml
new file mode 100644
index 0000000..43b59a6
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml
@@ -0,0 +1,5 @@
+- project:
+    name: org/project3
+    check:
+      jobs:
+        - restricted-job
diff --git a/tests/fixtures/config/allowed-projects/main.yaml b/tests/fixtures/config/allowed-projects/main.yaml
new file mode 100644
index 0000000..49ed838
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/main.yaml
@@ -0,0 +1,10 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2
+          - org/project3
diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml
new file mode 100644
index 0000000..9954846
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml
@@ -0,0 +1,22 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- project:
+    name: common-config
+    check:
+      jobs: []
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/README b/tests/fixtures/config/branch-mismatch/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..809f830
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml
@@ -0,0 +1,7 @@
+- job:
+    name: project-test1
+
+- project:
+    check:
+      jobs:
+        - project-test1
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/README b/tests/fixtures/config/branch-mismatch/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..3a8e9df
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml
@@ -0,0 +1,13 @@
+- job:
+    name: project-test2
+    parent: project-test1
+    override-checkout: stable
+
+- project:
+    check:
+      jobs:
+        - project-test1:
+            required-projects:
+              - name: org/project1
+                override-checkout: stable
+        - project-test2
diff --git a/tests/fixtures/config/branch-mismatch/main.yaml b/tests/fixtures/config/branch-mismatch/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2
diff --git a/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml b/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml
index 95ab870..ba35eb0 100644
--- a/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml
+++ b/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml
@@ -1,6 +1,7 @@
 - hosts: localhost
   tasks:
     - command: dd if=/dev/zero of=toobig bs=1M count=2
+    - command: sync
     - wait_for:
         delay: 10
         path: /
diff --git a/tests/fixtures/config/governor/git/common-config/playbooks/base.yaml b/tests/fixtures/config/governor/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/governor/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/governor/git/common-config/zuul.yaml b/tests/fixtures/config/governor/git/common-config/zuul.yaml
new file mode 100644
index 0000000..093da16
--- /dev/null
+++ b/tests/fixtures/config/governor/git/common-config/zuul.yaml
@@ -0,0 +1,34 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+    
+- job:
+    name: test1
+
+- job:
+    name: test2
+
+- job:
+    name: test3
+
+- project:
+    name: common-config
+    check:
+      jobs:
+        - test1
+        - test2
+        - test3
diff --git a/tests/fixtures/config/governor/main.yaml b/tests/fixtures/config/governor/main.yaml
new file mode 100644
index 0000000..9d01f54
--- /dev/null
+++ b/tests/fixtures/config/governor/main.yaml
@@ -0,0 +1,6 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
diff --git a/tests/fixtures/config/nodesets/git/common-config/playbooks/base.yaml b/tests/fixtures/config/nodesets/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/nodesets/git/common-config/zuul.yaml b/tests/fixtures/config/nodesets/git/common-config/zuul.yaml
new file mode 100644
index 0000000..e1e2fb7
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/common-config/zuul.yaml
@@ -0,0 +1,39 @@
+- pipeline:
+    name: check
+    manager: independent
+    post-review: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
diff --git a/tests/fixtures/config/nodesets/git/org_project1/README b/tests/fixtures/config/nodesets/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/nodesets/git/org_project1/zuul.yaml b/tests/fixtures/config/nodesets/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..398269e
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project1/zuul.yaml
@@ -0,0 +1,15 @@
+- nodeset:
+    name: project1-nodeset
+    nodes:
+      - name: controller
+        label: ubuntu-xenial
+
+- job:
+    parent: base
+    name: project1-test
+    nodeset: project1-nodeset
+
+- project:
+    check:
+      jobs:
+        - project1-test
diff --git a/tests/fixtures/config/nodesets/git/org_project2/README b/tests/fixtures/config/nodesets/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/nodesets/git/org_project2/zuul-nodeset.yaml b/tests/fixtures/config/nodesets/git/org_project2/zuul-nodeset.yaml
new file mode 100644
index 0000000..cb969a7
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project2/zuul-nodeset.yaml
@@ -0,0 +1,18 @@
+- nodeset:
+    name: project2-nodeset
+    nodes:
+      name: controller
+      label: ubuntu-xenial
+
+- job:
+    parent: base
+    name: project2-test
+    nodeset: project2-nodeset
+
+- project:
+    check:
+      jobs:
+        - project2-test
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/nodesets/git/org_project2/zuul.yaml b/tests/fixtures/config/nodesets/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..a4b42b1
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+    parent: base
+    name: project2-test
+
+- project:
+    check:
+      jobs:
+        - project2-test
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/nodesets/main.yaml b/tests/fixtures/config/nodesets/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/nodesets/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2
diff --git a/tests/fixtures/config/secrets/git/common-config/zuul.yaml b/tests/fixtures/config/secrets/git/common-config/zuul.yaml
new file mode 100644
index 0000000..f9dfacc
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/common-config/zuul.yaml
@@ -0,0 +1,38 @@
+- pipeline:
+    name: check
+    manager: independent
+    post-review: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
diff --git a/tests/fixtures/config/secrets/git/org_project1/README b/tests/fixtures/config/secrets/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/secrets/git/org_project1/playbooks/secret.yaml b/tests/fixtures/config/secrets/git/org_project1/playbooks/secret.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project1/playbooks/secret.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/secrets/git/org_project1/zuul.yaml b/tests/fixtures/config/secrets/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..f105ada
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project1/zuul.yaml
@@ -0,0 +1,26 @@
+- secret:
+    name: project1_secret
+    data:
+      username: test-username
+      password: !encrypted/pkcs1-oaep |
+        BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+        L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+        ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+        3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+        Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+        xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+        aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+        Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+        +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
+- job:
+    parent: base
+    name: project1-secret
+    run: playbooks/secret.yaml
+    secrets:
+      - project1_secret
+
+- project:
+    check:
+      jobs:
+        - project1-secret
diff --git a/tests/fixtures/config/secrets/git/org_project2/README b/tests/fixtures/config/secrets/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/secrets/git/org_project2/playbooks/secret.yaml b/tests/fixtures/config/secrets/git/org_project2/playbooks/secret.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/playbooks/secret.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/secrets/git/org_project2/zuul-secret.yaml b/tests/fixtures/config/secrets/git/org_project2/zuul-secret.yaml
new file mode 100644
index 0000000..d6ffd47
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/zuul-secret.yaml
@@ -0,0 +1,29 @@
+- secret:
+    name: project2_secret
+    data:
+      username: test-username
+      password: !encrypted/pkcs1-oaep |
+        BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+        L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+        ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+        3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+        Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+        xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+        aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+        Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+        +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
+- job:
+    parent: base
+    name: project2-secret
+    run: playbooks/secret.yaml
+    secrets:
+      - project2_secret
+
+- project:
+    check:
+      jobs:
+        - project2-secret
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/secrets/git/org_project2/zuul.yaml b/tests/fixtures/config/secrets/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..305a237
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/zuul.yaml
@@ -0,0 +1,12 @@
+- job:
+    parent: base
+    name: project2-secret
+    run: playbooks/secret.yaml
+
+- project:
+    check:
+      jobs:
+        - project2-secret
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/secrets/main.yaml b/tests/fixtures/config/secrets/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/secrets/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2
diff --git a/tests/fixtures/config/semaphore-branches/git/common-config/playbooks/base.yaml b/tests/fixtures/config/semaphore-branches/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/semaphore-branches/git/common-config/zuul.yaml b/tests/fixtures/config/semaphore-branches/git/common-config/zuul.yaml
new file mode 100644
index 0000000..e1e2fb7
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/common-config/zuul.yaml
@@ -0,0 +1,39 @@
+- pipeline:
+    name: check
+    manager: independent
+    post-review: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project1/README b/tests/fixtures/config/semaphore-branches/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project1/zuul.yaml b/tests/fixtures/config/semaphore-branches/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..73766e0
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project1/zuul.yaml
@@ -0,0 +1,13 @@
+- semaphore:
+    name: project1-semaphore
+    max: 2
+
+- job:
+    parent: base
+    name: project1-test
+    semaphore: project1-semaphore
+
+- project:
+    check:
+      jobs:
+        - project1-test
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project2/README b/tests/fixtures/config/semaphore-branches/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project2/zuul-semaphore.yaml b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul-semaphore.yaml
new file mode 100644
index 0000000..db93fdb
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul-semaphore.yaml
@@ -0,0 +1,16 @@
+- semaphore:
+    name: project2-semaphore
+    max: 2
+
+- job:
+    parent: base
+    name: project2-test
+    semaphore: project2-semaphore
+
+- project:
+    check:
+      jobs:
+        - project2-test
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project2/zuul.yaml b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..a4b42b1
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+    parent: base
+    name: project2-test
+
+- project:
+    check:
+      jobs:
+        - project2-test
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/semaphore-branches/main.yaml b/tests/fixtures/config/semaphore-branches/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2
diff --git a/tests/fixtures/git_fetch_error.sh b/tests/fixtures/git_fetch_error.sh
new file mode 100755
index 0000000..49c568c
--- /dev/null
+++ b/tests/fixtures/git_fetch_error.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+echo $*
+case "$1" in
+    fetch)
+	if [ -f ./stamp1 ]; then
+	    touch ./stamp2
+	    exit 0
+	fi
+	touch ./stamp1
+	exit 1
+	;;
+    version)
+        echo "git version 1.0.0"
+        exit 0
+        ;;
+esac
diff --git a/tests/fixtures/layouts/branch-deletion.yaml b/tests/fixtures/layouts/branch-deletion.yaml
new file mode 100644
index 0000000..f72902a
--- /dev/null
+++ b/tests/fixtures/layouts/branch-deletion.yaml
@@ -0,0 +1,34 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: project-test1
+    parent: base
+    branches: master
+
+- job:
+    name: project-test2
+    parent: base
+    branches: stable
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - project-test1
+        - project-test2
diff --git a/tests/fixtures/layouts/timer-github.yaml b/tests/fixtures/layouts/timer-github.yaml
new file mode 100644
index 0000000..4f3efe4
--- /dev/null
+++ b/tests/fixtures/layouts/timer-github.yaml
@@ -0,0 +1,25 @@
+- pipeline:
+    name: periodic
+    manager: independent
+    trigger:
+      timer:
+        - time: '* * * * * */1'
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: project-bitrot
+    nodeset:
+      nodes:
+        - name: static
+          label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
+
+- project:
+    name: org/project
+    periodic:
+      jobs:
+        - project-bitrot
diff --git a/tests/fixtures/zuul-connections-merger.conf b/tests/fixtures/zuul-connections-merger.conf
index 771fc50..15769ef 100644
--- a/tests/fixtures/zuul-connections-merger.conf
+++ b/tests/fixtures/zuul-connections-merger.conf
@@ -1,7 +1,7 @@
 [gearman]
 server=127.0.0.1
 
-[webapp]
+[web]
 status_url=http://zuul.example.com/status
 
 [merger]
diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf
index a96bde2..b6a7753 100644
--- a/tests/fixtures/zuul-github-driver.conf
+++ b/tests/fixtures/zuul-github-driver.conf
@@ -1,7 +1,7 @@
 [gearman]
 server=127.0.0.1
 
-[webapp]
+[web]
 status_url=http://zuul.example.com/status/#{change.number},{change.patchset}
 
 [merger]
diff --git a/tests/fixtures/zuul-push-reqs.conf b/tests/fixtures/zuul-push-reqs.conf
index 2217f94..b902d3f 100644
--- a/tests/fixtures/zuul-push-reqs.conf
+++ b/tests/fixtures/zuul-push-reqs.conf
@@ -1,7 +1,7 @@
 [gearman]
 server=127.0.0.1
 
-[webapp]
+[web]
 status_url=http://zuul.example.com/status
 
 [merger]
diff --git a/tests/unit/test_disk_accountant.py b/tests/unit/test_disk_accountant.py
index 7081b53..e12846d 100644
--- a/tests/unit/test_disk_accountant.py
+++ b/tests/unit/test_disk_accountant.py
@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import fixtures
 import os
 import tempfile
 import time
@@ -32,6 +33,10 @@
 
 
 class TestDiskAccountant(BaseTestCase):
+    def setUp(self):
+        super(TestDiskAccountant, self).setUp()
+        self.useFixture(fixtures.NestedTempfile())
+
     def test_disk_accountant(self):
         jobs_dir = tempfile.mkdtemp(
             dir=os.environ.get("ZUUL_TEST_ROOT", None))
@@ -47,6 +52,8 @@
             testfile = os.path.join(jobdir, 'tfile')
             with open(testfile, 'w') as tf:
                 tf.write(2 * 1024 * 1024 * '.')
+                tf.flush()
+                os.fsync(tf.fileno())
 
             # da should catch over-limit dir within 5 seconds
             for i in range(0, 50):
diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py
index b424769..0a5c0a4 100644
--- a/tests/unit/test_encryption.py
+++ b/tests/unit/test_encryption.py
@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import fixtures
 import os
 import subprocess
 import tempfile
@@ -26,6 +27,10 @@
     def setUp(self):
         super(TestEncryption, self).setUp()
         self.private, self.public = encryption.generate_rsa_keypair()
+        # Because we set delete to False when using NamedTemporaryFile below
+        # we need to stick our usage of temporary files in the NestedTempfile
+        # fixture ensuring everything gets cleaned up when it is done.
+        self.useFixture(fixtures.NestedTempfile())
 
     def test_serialization(self):
         "Verify key serialization"
diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py
index 474859d..46e1d99 100755
--- a/tests/unit/test_executor.py
+++ b/tests/unit/test_executor.py
@@ -15,6 +15,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
 import logging
 import time
 
@@ -436,3 +441,78 @@
     def test_executor_hostname(self):
         self.assertEqual('test-executor-hostname.example.com',
                          self.executor_server.hostname)
+
+
+class TestGovernor(ZuulTestCase):
+    tenant_config_file = 'config/governor/main.yaml'
+
+    @mock.patch('os.getloadavg')
+    @mock.patch('psutil.virtual_memory')
+    def test_load_governor(self, vm_mock, loadavg_mock):
+        class Dummy(object):
+            pass
+        ram = Dummy()
+        ram.percent = 20.0  # 20% used
+        vm_mock.return_value = ram
+        loadavg_mock.return_value = (0.0, 0.0, 0.0)
+        self.executor_server.manageLoad()
+        self.assertTrue(self.executor_server.accepting_work)
+        ram.percent = 99.0  # 99% used
+        loadavg_mock.return_value = (100.0, 100.0, 100.0)
+        self.executor_server.manageLoad()
+        self.assertFalse(self.executor_server.accepting_work)
+
+    def waitForExecutorBuild(self, jobname):
+        timeout = time.time() + 30
+        build = None
+        while (time.time() < timeout and not build):
+            for b in self.builds:
+                if b.name == jobname:
+                    build = b
+                    break
+            time.sleep(0.1)
+        build_id = build.uuid
+        while (time.time() < timeout and
+               build_id not in self.executor_server.job_workers):
+            time.sleep(0.1)
+        worker = self.executor_server.job_workers[build_id]
+        while (time.time() < timeout and
+               not worker.started):
+            time.sleep(0.1)
+        return build
+
+    def waitForWorkerCompletion(self, build):
+        timeout = time.time() + 30
+        while (time.time() < timeout and
+               build.uuid in self.executor_server.job_workers):
+            time.sleep(0.1)
+
+    def test_slow_start(self):
+        self.executor_server.hold_jobs_in_build = True
+        self.executor_server.max_starting_builds = 1
+        self.executor_server.min_starting_builds = 1
+        self.executor_server.manageLoad()
+        self.assertTrue(self.executor_server.accepting_work)
+        A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+
+        build1 = self.waitForExecutorBuild('test1')
+        # With one job (test1) being started, we should no longer
+        # be accepting new work
+        self.assertFalse(self.executor_server.accepting_work)
+        self.assertEqual(len(self.executor_server.job_workers), 1)
+        # Allow enough starting builds for the test to complete.
+        self.executor_server.max_starting_builds = 3
+        build1.release()
+        self.waitForWorkerCompletion(build1)
+        self.executor_server.manageLoad()
+
+        self.waitForExecutorBuild('test2')
+        self.waitForExecutorBuild('test3')
+        self.assertFalse(self.executor_server.accepting_work)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+        self.executor_server.manageLoad()
+        self.assertTrue(self.executor_server.accepting_work)
diff --git a/tests/unit/test_gerrit_crd.py b/tests/unit/test_gerrit_crd.py
index a8924b9..ad25c47 100644
--- a/tests/unit/test_gerrit_crd.py
+++ b/tests/unit/test_gerrit_crd.py
@@ -54,8 +54,11 @@
         A.setDependsOn(AM1, 1)
         AM1.setDependsOn(AM2, 1)
 
+        # So that at least one test uses the /#/c/ form of the url,
+        # use it here.
+        url = 'https://%s/#/c/%s' % (B.gerrit.server, B.number)
         A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
-            A.subject, B.data['url'])
+            A.subject, url)
 
         self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
         self.waitUntilSettled()
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 3942b0b..8978415 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -12,15 +12,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import asyncio
+import threading
 import os
 import re
 from testtools.matchers import MatchesRegex, StartsWith
 import urllib
+import socket
 import time
 from unittest import skip
 
 import git
 
+import zuul.web
+
 from tests.base import ZuulTestCase, simple_layout, random_sha1
 
 
@@ -205,6 +210,34 @@
         self.waitUntilSettled()
         self.assertEqual(1, len(self.history))
 
+    @simple_layout('layouts/basic-github.yaml', driver='github')
+    def test_timer_event(self):
+        self.executor_server.hold_jobs_in_build = True
+        self.commitConfigUpdate('org/common-config',
+                                'layouts/timer-github.yaml')
+        self.sched.reconfigure(self.config)
+        time.sleep(2)
+        self.waitUntilSettled()
+        self.assertEqual(len(self.builds), 1)
+        self.executor_server.hold_jobs_in_build = False
+        # Stop queuing timer triggered jobs so that the assertions
+        # below don't race against more jobs being queued.
+        self.commitConfigUpdate('org/common-config',
+                                'layouts/basic-github.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+        # If APScheduler is in mid-event when we remove the job, we
+        # can end up with one more event firing, so give it an extra
+        # second to settle.
+        time.sleep(1)
+        self.waitUntilSettled()
+        self.executor_server.release()
+        self.waitUntilSettled()
+        self.assertHistory([
+            dict(name='project-bitrot', result='SUCCESS',
+                 ref='refs/heads/master'),
+        ], ordered=False)
+
     @simple_layout('layouts/dequeue-github.yaml', driver='github')
     def test_dequeue_pull_synchronized(self):
         self.executor_server.hold_jobs_in_build = True
@@ -734,3 +767,85 @@
 
         # project2 should have no parsed branch
         self.assertEqual(0, len(project2.unparsed_branch_config.keys()))
+
+
+class TestGithubWebhook(ZuulTestCase):
+    config_file = 'zuul-github-driver.conf'
+
+    def setUp(self):
+        super(TestGithubWebhook, self).setUp()
+
+        # Start the web server
+        self.web = zuul.web.ZuulWeb(
+            listen_address='127.0.0.1', listen_port=0,
+            gear_server='127.0.0.1', gear_port=self.gearman_server.port,
+            connections=[self.fake_github])
+        loop = asyncio.new_event_loop()
+        loop.set_debug(True)
+        ws_thread = threading.Thread(target=self.web.run, args=(loop,))
+        ws_thread.start()
+        self.addCleanup(loop.close)
+        self.addCleanup(ws_thread.join)
+        self.addCleanup(self.web.stop)
+
+        host = '127.0.0.1'
+        # Wait until web server is started
+        while True:
+            time.sleep(0.1)
+            if self.web.server is None:
+                continue
+            port = self.web.server.sockets[0].getsockname()[1]
+            try:
+                with socket.create_connection((host, port)):
+                    break
+            except ConnectionRefusedError:
+                pass
+
+        self.fake_github.setZuulWebPort(port)
+
+    def tearDown(self):
+        super(TestGithubWebhook, self).tearDown()
+
+    @simple_layout('layouts/basic-github.yaml', driver='github')
+    def test_webhook(self):
+        """Test that we can get github events via zuul-web."""
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+        self.fake_github.emitEvent(A.getPullRequestOpenedEvent(),
+                                   use_zuulweb=True)
+        self.waitUntilSettled()
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual('SUCCESS',
+                         self.getJobFromHistory('project-test1').result)
+        self.assertEqual('SUCCESS',
+                         self.getJobFromHistory('project-test2').result)
+
+        job = self.getJobFromHistory('project-test2')
+        zuulvars = job.parameters['zuul']
+        self.assertEqual(str(A.number), zuulvars['change'])
+        self.assertEqual(str(A.head_sha), zuulvars['patchset'])
+        self.assertEqual('master', zuulvars['branch'])
+        self.assertEqual(1, len(A.comments))
+        self.assertThat(
+            A.comments[0],
+            MatchesRegex('.*\[project-test1 \]\(.*\).*', re.DOTALL))
+        self.assertThat(
+            A.comments[0],
+            MatchesRegex('.*\[project-test2 \]\(.*\).*', re.DOTALL))
+        self.assertEqual(2, len(self.history))
+
+        # test_pull_unmatched_branch_event(self):
+        self.create_branch('org/project', 'unmatched_branch')
+        B = self.fake_github.openFakePullRequest(
+            'org/project', 'unmatched_branch', 'B')
+        self.fake_github.emitEvent(B.getPullRequestOpenedEvent(),
+                                   use_zuulweb=True)
+        self.waitUntilSettled()
+
+        self.assertEqual(2, len(self.history))
diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py
index ec30a2b..fb2f199 100644
--- a/tests/unit/test_merger_repo.py
+++ b/tests/unit/test_merger_repo.py
@@ -82,7 +82,7 @@
                    os.path.join(FIXTURE_DIR, 'fake_git.sh'))
         work_repo = Repo(parent_path, self.workspace_root,
                          'none@example.org', 'User Name', '0', '0',
-                         git_timeout=0.001)
+                         git_timeout=0.001, retry_attempts=1)
         # TODO: have the merger and repo classes catch fewer
         # exceptions, including this one on initialization.  For the
         # test, we try cloning again.
@@ -93,10 +93,26 @@
     def test_fetch_timeout(self):
         parent_path = os.path.join(self.upstream_root, 'org/project1')
         work_repo = Repo(parent_path, self.workspace_root,
-                         'none@example.org', 'User Name', '0', '0')
+                         'none@example.org', 'User Name', '0', '0',
+                         retry_attempts=1)
         work_repo.git_timeout = 0.001
         self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',
                    os.path.join(FIXTURE_DIR, 'fake_git.sh'))
         with testtools.ExpectedException(git.exc.GitCommandError,
                                          '.*exit code\(-9\)'):
             work_repo.update()
+
+    def test_fetch_retry(self):
+        parent_path = os.path.join(self.upstream_root, 'org/project1')
+        work_repo = Repo(parent_path, self.workspace_root,
+                         'none@example.org', 'User Name', '0', '0',
+                         retry_interval=1)
+        self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',
+                   os.path.join(FIXTURE_DIR, 'git_fetch_error.sh'))
+        work_repo.update()
+        # This is created on the first fetch
+        self.assertTrue(os.path.exists(os.path.join(
+            self.workspace_root, 'stamp1')))
+        # This is created on the second fetch
+        self.assertTrue(os.path.exists(os.path.join(
+            self.workspace_root, 'stamp2')))
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 784fcb3..5c586ca 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -320,50 +320,6 @@
                 "to shadow job base in base_project"):
             layout.addJob(base2)
 
-    def test_job_allowed_projects(self):
-        job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
-            '_source_context': self.context,
-            '_start_mark': self.start_mark,
-            'name': 'job',
-            'parent': None,
-            'allowed-projects': ['project'],
-        })
-        self.layout.addJob(job)
-
-        project2 = model.Project('project2', self.source)
-        tpc2 = model.TenantProjectConfig(project2)
-        self.tenant.addUntrustedProject(tpc2)
-        context2 = model.SourceContext(project2, 'master',
-                                       'test', True)
-
-        project_template_parser = configloader.ProjectTemplateParser(
-            self.tenant, self.layout)
-        project_parser = configloader.ProjectParser(
-            self.tenant, self.layout, project_template_parser)
-        project2_config = project_parser.fromYaml(
-            [{
-                '_source_context': context2,
-                '_start_mark': self.start_mark,
-                'name': 'project2',
-                'gate': {
-                    'jobs': [
-                        'job'
-                    ]
-                }
-            }]
-        )
-        self.layout.addProjectConfig(project2_config)
-
-        change = model.Change(project2)
-        # Test master
-        change.branch = 'master'
-        item = self.queue.enqueueChange(change)
-        item.layout = self.layout
-        with testtools.ExpectedException(
-                Exception,
-                "Project project2 is not allowed to run job job"):
-            item.freezeJobGraph()
-
     def test_job_pipeline_allow_untrusted_secrets(self):
         self.pipeline.post_review = False
         job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 5db20b3..c833fa2 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -19,14 +19,12 @@
 import textwrap
 
 import os
-import re
 import shutil
 import time
 from unittest import skip
 
 import git
 import testtools
-import urllib
 
 import zuul.change_matcher
 from zuul.driver.gerrit import gerritreporter
@@ -39,6 +37,7 @@
     ZuulTestCase,
     repack_repo,
     simple_layout,
+    iterate_timeout,
 )
 
 
@@ -161,6 +160,44 @@
         self.assertEqual(self.getJobFromHistory('project-test1').node,
                          'label2')
 
+    @simple_layout('layouts/branch-deletion.yaml')
+    def test_branch_deletion(self):
+        "Test the correct variant of a job runs on a branch"
+        self._startMerger()
+        for f in list(self.executor_server.merger_worker.functions.keys()):
+            f = str(f)
+            if f.startswith('merger:'):
+                self.executor_server.merger_worker.unRegisterFunction(f)
+
+        self.create_branch('org/project', 'stable')
+        A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(self.getJobFromHistory('project-test2').result,
+                         'SUCCESS')
+
+        self.delete_branch('org/project', 'stable')
+        path = os.path.join(self.executor_src_root, 'review.example.com')
+        shutil.rmtree(path)
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        build = self.builds[0]
+
+        # Make sure there is no stable branch in the checked out git repo.
+        pname = 'review.example.com/org/project'
+        work = build.getWorkspaceRepos([pname])
+        work = work[pname]
+        heads = set([str(x) for x in work.heads])
+        self.assertEqual(heads, set(['master']))
+        self.executor_server.hold_jobs_in_build = False
+        build.release()
+        self.waitUntilSettled()
+        self.assertEqual(self.getJobFromHistory('project-test1').result,
+                         'SUCCESS')
+
     def test_parallel_changes(self):
         "Test that changes are tested in parallel and merged in series"
 
@@ -1469,7 +1506,7 @@
                                           self.gearman_server.port)
         self.addCleanup(client.shutdown)
         r = client.autohold('tenant-one', 'org/project', 'project-test2',
-                            "reason text", 1)
+                            "", "", "reason text", 1)
         self.assertTrue(r)
 
         # First check that successful jobs do not autohold
@@ -1516,7 +1553,7 @@
             held_node['hold_job'],
             " ".join(['tenant-one',
                       'review.example.com/org/project',
-                      'project-test2'])
+                      'project-test2', '.*'])
         )
         self.assertEqual(held_node['comment'], "reason text")
 
@@ -1536,13 +1573,151 @@
                 held_nodes += 1
         self.assertEqual(held_nodes, 1)
 
+    def _test_autohold_scoped(self, change_obj, change, ref):
+        client = zuul.rpcclient.RPCClient('127.0.0.1',
+                                          self.gearman_server.port)
+        self.addCleanup(client.shutdown)
+
+        # create two changes on the same project, and autohold request
+        # for one of them.
+        other = self.fake_gerrit.addFakeChange(
+            'org/project', 'master', 'other'
+        )
+
+        r = client.autohold('tenant-one', 'org/project', 'project-test2',
+                            str(change), ref, "reason text", 1)
+        self.assertTrue(r)
+
+        # First, check that an unrelated job does not trigger autohold, even
+        # when it failed
+        self.executor_server.failJob('project-test2', other)
+        self.fake_gerrit.addEvent(other.getPatchsetCreatedEvent(1))
+
+        self.waitUntilSettled()
+
+        self.assertEqual(other.data['status'], 'NEW')
+        self.assertEqual(other.reported, 1)
+        # project-test2
+        self.assertEqual(self.history[0].result, 'FAILURE')
+
+        # Check nodepool for a held node
+        held_node = None
+        for node in self.fake_nodepool.getNodes():
+            if node['state'] == zuul.model.STATE_HOLD:
+                held_node = node
+                break
+        self.assertIsNone(held_node)
+
+        # And then verify that failed job for the defined change
+        # triggers the autohold
+
+        self.executor_server.failJob('project-test2', change_obj)
+        self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))
+
+        self.waitUntilSettled()
+
+        self.assertEqual(change_obj.data['status'], 'NEW')
+        self.assertEqual(change_obj.reported, 1)
+        # project-test2
+        self.assertEqual(self.history[1].result, 'FAILURE')
+
+        # Check nodepool for a held node
+        held_node = None
+        for node in self.fake_nodepool.getNodes():
+            if node['state'] == zuul.model.STATE_HOLD:
+                held_node = node
+                break
+        self.assertIsNotNone(held_node)
+
+        # Validate node has recorded the failed job
+        if change != "":
+            ref = "refs/changes/%s/%s/.*" % (
+                str(change_obj.number)[-1:], str(change_obj.number)
+            )
+
+        self.assertEqual(
+            held_node['hold_job'],
+            " ".join(['tenant-one',
+                      'review.example.com/org/project',
+                      'project-test2', ref])
+        )
+        self.assertEqual(held_node['comment'], "reason text")
+
+    @simple_layout('layouts/autohold.yaml')
+    def test_autohold_change(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+
+        self._test_autohold_scoped(A, change=A.number, ref="")
+
+    @simple_layout('layouts/autohold.yaml')
+    def test_autohold_ref(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        ref = A.data['currentPatchSet']['ref']
+        self._test_autohold_scoped(A, change="", ref=ref)
+
+    @simple_layout('layouts/autohold.yaml')
+    def test_autohold_scoping(self):
+        client = zuul.rpcclient.RPCClient('127.0.0.1',
+                                          self.gearman_server.port)
+        self.addCleanup(client.shutdown)
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+
+        # create three autohold requests, scoped to job, change and
+        # a specific ref
+        change = str(A.number)
+        ref = A.data['currentPatchSet']['ref']
+        r1 = client.autohold('tenant-one', 'org/project', 'project-test2',
+                             "", "", "reason text", 1)
+        self.assertTrue(r1)
+        r2 = client.autohold('tenant-one', 'org/project', 'project-test2',
+                             change, "", "reason text", 1)
+        self.assertTrue(r2)
+        r3 = client.autohold('tenant-one', 'org/project', 'project-test2',
+                             "", ref, "reason text", 1)
+        self.assertTrue(r3)
+
+        # Fail 3 jobs for the same change, and verify that the autohold
+        # requests are fullfilled in the expected order: from the most
+        # specific towards the most generic one.
+
+        def _fail_job_and_verify_autohold_request(change_obj, ref_filter):
+            self.executor_server.failJob('project-test2', change_obj)
+            self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))
+
+            self.waitUntilSettled()
+
+            # Check nodepool for a held node
+            held_node = None
+            for node in self.fake_nodepool.getNodes():
+                if node['state'] == zuul.model.STATE_HOLD:
+                    held_node = node
+                    break
+            self.assertIsNotNone(held_node)
+
+            self.assertEqual(
+                held_node['hold_job'],
+                " ".join(['tenant-one',
+                          'review.example.com/org/project',
+                          'project-test2', ref_filter])
+            )
+            self.assertFalse(held_node['_lock'], "Node %s is locked" %
+                             (node['_oid'],))
+            self.fake_nodepool.removeNode(held_node)
+
+        _fail_job_and_verify_autohold_request(A, ref)
+
+        ref = "refs/changes/%s/%s/.*" % (str(change)[-1:], str(change))
+        _fail_job_and_verify_autohold_request(A, ref)
+        _fail_job_and_verify_autohold_request(A, ".*")
+
     @simple_layout('layouts/autohold.yaml')
     def test_autohold_ignores_aborted_jobs(self):
         client = zuul.rpcclient.RPCClient('127.0.0.1',
                                           self.gearman_server.port)
         self.addCleanup(client.shutdown)
         r = client.autohold('tenant-one', 'org/project', 'project-test2',
-                            "reason text", 1)
+                            "", "", "reason text", 1)
         self.assertTrue(r)
 
         self.executor_server.hold_jobs_in_build = True
@@ -1586,7 +1761,7 @@
         self.addCleanup(client.shutdown)
 
         r = client.autohold('tenant-one', 'org/project', 'project-test2',
-                            "reason text", 1)
+                            "", "", "reason text", 1)
         self.assertTrue(r)
 
         autohold_requests = client.autohold_list()
@@ -1595,11 +1770,12 @@
 
         # The single dict key should be a CSV string value
         key = list(autohold_requests.keys())[0]
-        tenant, project, job = key.split(',')
+        tenant, project, job, ref_filter = key.split(',')
 
         self.assertEqual('tenant-one', tenant)
         self.assertIn('org/project', project)
         self.assertEqual('project-test2', job)
+        self.assertEqual(".*", ref_filter)
 
         # Note: the value is converted from set to list by json.
         self.assertEqual([1, "reason text"], autohold_requests[key])
@@ -2533,110 +2709,6 @@
         self.assertEqual(self.history[4].pipeline, 'check')
         self.assertEqual(self.history[5].pipeline, 'check')
 
-    def test_json_status(self):
-        "Test that we can retrieve JSON status info"
-        self.executor_server.hold_jobs_in_build = True
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        A.addApproval('Code-Review', 2)
-        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
-        self.waitUntilSettled()
-
-        self.executor_server.release('project-merge')
-        self.waitUntilSettled()
-
-        port = self.webapp.server.socket.getsockname()[1]
-
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/status" % port)
-        f = urllib.request.urlopen(req)
-        headers = f.info()
-        self.assertIn('Content-Length', headers)
-        self.assertIn('Content-Type', headers)
-        self.assertIsNotNone(re.match('^application/json(; charset=UTF-8)?$',
-                                      headers['Content-Type']))
-        self.assertIn('Access-Control-Allow-Origin', headers)
-        self.assertIn('Cache-Control', headers)
-        self.assertIn('Last-Modified', headers)
-        self.assertIn('Expires', headers)
-        data = f.read().decode('utf8')
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-        self.waitUntilSettled()
-
-        data = json.loads(data)
-        status_jobs = []
-        for p in data['pipelines']:
-            for q in p['change_queues']:
-                if p['name'] in ['gate', 'conflict']:
-                    self.assertEqual(q['window'], 20)
-                else:
-                    self.assertEqual(q['window'], 0)
-                for head in q['heads']:
-                    for change in head:
-                        self.assertTrue(change['active'])
-                        self.assertEqual(change['id'], '1,1')
-                        for job in change['jobs']:
-                            status_jobs.append(job)
-        self.assertEqual('project-merge', status_jobs[0]['name'])
-        # TODO(mordred) pull uuids from self.builds
-        self.assertEqual(
-            'stream.html?uuid={uuid}&logfile=console.log'.format(
-                uuid=status_jobs[0]['uuid']),
-            status_jobs[0]['url'])
-        self.assertEqual(
-            'finger://{hostname}/{uuid}'.format(
-                hostname=self.executor_server.hostname,
-                uuid=status_jobs[0]['uuid']),
-            status_jobs[0]['finger_url'])
-        # TOOD(mordred) configure a success-url on the base job
-        self.assertEqual(
-            'finger://{hostname}/{uuid}'.format(
-                hostname=self.executor_server.hostname,
-                uuid=status_jobs[0]['uuid']),
-            status_jobs[0]['report_url'])
-        self.assertEqual('project-test1', status_jobs[1]['name'])
-        self.assertEqual(
-            'stream.html?uuid={uuid}&logfile=console.log'.format(
-                uuid=status_jobs[1]['uuid']),
-            status_jobs[1]['url'])
-        self.assertEqual(
-            'finger://{hostname}/{uuid}'.format(
-                hostname=self.executor_server.hostname,
-                uuid=status_jobs[1]['uuid']),
-            status_jobs[1]['finger_url'])
-        self.assertEqual(
-            'finger://{hostname}/{uuid}'.format(
-                hostname=self.executor_server.hostname,
-                uuid=status_jobs[1]['uuid']),
-            status_jobs[1]['report_url'])
-
-        self.assertEqual('project-test2', status_jobs[2]['name'])
-        self.assertEqual(
-            'stream.html?uuid={uuid}&logfile=console.log'.format(
-                uuid=status_jobs[2]['uuid']),
-            status_jobs[2]['url'])
-        self.assertEqual(
-            'finger://{hostname}/{uuid}'.format(
-                hostname=self.executor_server.hostname,
-                uuid=status_jobs[2]['uuid']),
-            status_jobs[2]['finger_url'])
-        self.assertEqual(
-            'finger://{hostname}/{uuid}'.format(
-                hostname=self.executor_server.hostname,
-                uuid=status_jobs[2]['uuid']),
-            status_jobs[2]['report_url'])
-
-        # check job dependencies
-        self.assertIsNotNone(status_jobs[0]['dependencies'])
-        self.assertIsNotNone(status_jobs[1]['dependencies'])
-        self.assertIsNotNone(status_jobs[2]['dependencies'])
-        self.assertEqual(len(status_jobs[0]['dependencies']), 0)
-        self.assertEqual(len(status_jobs[1]['dependencies']), 1)
-        self.assertEqual(len(status_jobs[2]['dependencies']), 1)
-        self.assertIn('project-merge', status_jobs[1]['dependencies'])
-        self.assertIn('project-merge', status_jobs[2]['dependencies'])
-
     def test_reconfigure_merge(self):
         """Test that two reconfigure events are merged"""
 
@@ -3212,13 +3284,6 @@
 
         self.assertEqual(len(self.builds), 2)
 
-        port = self.webapp.server.socket.getsockname()[1]
-
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/status" % port)
-        f = urllib.request.urlopen(req)
-        data = f.read().decode('utf8')
-
         self.executor_server.hold_jobs_in_build = False
         # Stop queuing timer triggered jobs so that the assertions
         # below don't race against more jobs being queued.
@@ -3240,16 +3305,6 @@
                  ref='refs/heads/stable'),
         ], ordered=False)
 
-        data = json.loads(data)
-        status_jobs = set()
-        for p in data['pipelines']:
-            for q in p['change_queues']:
-                for head in q['heads']:
-                    for change in head:
-                        for job in change['jobs']:
-                            status_jobs.add(job['name'])
-        self.assertIn('project-bitrot', status_jobs)
-
     def test_idle(self):
         "Test that frequent periodic jobs work"
         # This test can not use simple_layout because it must start
@@ -4482,6 +4537,54 @@
         self.assertEqual(A.data['status'], 'MERGED')
         self.assertEqual(A.reported, 2)
 
+    def test_zookeeper_disconnect2(self):
+        "Test that jobs are executed after a zookeeper disconnect"
+
+        # This tests receiving a ZK disconnect between the arrival of
+        # a fulfilled request and when we accept its nodes.
+        self.fake_nodepool.paused = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+
+        # We're waiting on the nodepool request to complete.  Stop the
+        # scheduler from processing further events, then fulfill the
+        # nodepool request.
+        self.sched.run_handler_lock.acquire()
+
+        # Fulfill the nodepool request.
+        self.fake_nodepool.paused = False
+        requests = list(self.sched.nodepool.requests.values())
+        self.assertEqual(1, len(requests))
+        request = requests[0]
+        for x in iterate_timeout(30, 'fulfill request'):
+            if request.fulfilled:
+                break
+        id1 = request.id
+
+        # The request is fulfilled, but the scheduler hasn't processed
+        # it yet.  Reconnect ZK.
+        self.zk.client.stop()
+        self.zk.client.start()
+
+        # Allow the scheduler to continue and process the (now
+        # out-of-date) notification that nodes are ready.
+        self.sched.run_handler_lock.release()
+
+        # It should resubmit the request, once it's fulfilled, we can
+        # wait for it to run jobs and settle.
+        for x in iterate_timeout(30, 'fulfill request'):
+            if request.fulfilled:
+                break
+        self.waitUntilSettled()
+
+        id2 = request.id
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.assertEqual(A.reported, 2)
+        # Make sure it was resubmitted (the id's should be different).
+        self.assertNotEqual(id1, id2)
+
     def test_nodepool_failure(self):
         "Test that jobs are reported after a nodepool failure"
 
@@ -4934,7 +5037,7 @@
         return repo_messages
 
     def _test_merge(self, mode):
-        us_path = os.path.join(
+        us_path = 'file://' + os.path.join(
             self.upstream_root, 'org/project-%s' % mode)
         expected_messages = [
             'initial commit',
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 163a58b..44eda82 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -497,6 +497,72 @@
         self.waitUntilSettled()
 
 
+class TestBranchMismatch(ZuulTestCase):
+    tenant_config_file = 'config/branch-mismatch/main.yaml'
+
+    def test_job_override_branch(self):
+        "Test that override-checkout overrides branch matchers as well"
+
+        # Make sure the parent job repo is branched, so it gets
+        # implied branch matchers.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+
+        # The child job repo should have a branch which does not exist
+        # in the parent job repo.
+        self.create_branch('org/project2', 'devel')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project2', 'devel'))
+
+        # A job in a repo with a weird branch name should use the
+        # parent job from the parent job's master (default) branch.
+        A = self.fake_gerrit.addFakeChange('org/project2', 'devel', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        # project-test2 should run because it inherits from
+        # project-test1 and we will use the fallback branch to find
+        # project-test1 variants, but project-test1 itself, even
+        # though it is in the project-pipeline config, should not run
+        # because it doesn't directly match.
+        self.assertHistory([
+            dict(name='project-test1', result='SUCCESS', changes='1,1'),
+            dict(name='project-test2', result='SUCCESS', changes='1,1'),
+        ], ordered=False)
+
+
+class TestAllowedProjects(ZuulTestCase):
+    tenant_config_file = 'config/allowed-projects/main.yaml'
+
+    def test_allowed_projects(self):
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 1)
+        self.assertIn('Build succeeded', A.messages[0])
+
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(B.reported, 1)
+        self.assertIn('Project org/project2 is not allowed '
+                      'to run job test-project2', B.messages[0])
+
+        C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(C.reported, 1)
+        self.assertIn('Project org/project3 is not allowed '
+                      'to run job restricted-job', C.messages[0])
+
+        self.assertHistory([
+            dict(name='test-project1', result='SUCCESS', changes='1,1'),
+            dict(name='restricted-job', result='SUCCESS', changes='1,1'),
+        ], ordered=False)
+
+
 class TestCentralJobs(ZuulTestCase):
     tenant_config_file = 'config/central-jobs/main.yaml'
 
@@ -2528,6 +2594,157 @@
         self.assertHistory([])
 
 
+class TestSecrets(ZuulTestCase):
+    tenant_config_file = 'config/secrets/main.yaml'
+    secret = {'password': 'test-password',
+              'username': 'test-username'}
+
+    def _getSecrets(self, job, pbtype):
+        secrets = []
+        build = self.getJobFromHistory(job)
+        for pb in build.parameters[pbtype]:
+            secrets.append(pb['secrets'])
+        return secrets
+
+    def test_secret_branch(self):
+        # Test that we can use a secret defined in another branch of
+        # the same project.
+        self.create_branch('org/project2', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project2', 'stable'))
+        self.waitUntilSettled()
+
+        with open(os.path.join(FIXTURE_DIR,
+                               'config/secrets/git/',
+                               'org_project2/zuul-secret.yaml')) as f:
+            config = f.read()
+
+        file_dict = {'zuul.yaml': config}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                parent: base
+                name: project2-secret
+                run: playbooks/secret.yaml
+                secrets: [project2_secret]
+
+            - project:
+                check:
+                  jobs:
+                    - project2-secret
+                gate:
+                  jobs:
+                    - noop
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        B = self.fake_gerrit.addFakeChange('org/project2', 'stable', 'B',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(B.reported, 1, "B should report success")
+        self.assertHistory([
+            dict(name='project2-secret', result='SUCCESS', changes='2,1'),
+        ])
+        self.assertEqual(
+            self._getSecrets('project2-secret', 'playbooks'),
+            [{'project2_secret': self.secret}])
+
+    def test_secret_branch_duplicate(self):
+        # Test that we can create a duplicate secret on a different
+        # branch of the same project -- i.e., that when we branch
+        # master to stable on a project with a secret, nothing
+        # changes.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+        self.waitUntilSettled()
+
+        A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 1,
+                         "A should report success")
+        self.assertHistory([
+            dict(name='project1-secret', result='SUCCESS', changes='1,1'),
+        ])
+        self.assertEqual(
+            self._getSecrets('project1-secret', 'playbooks'),
+            [{'project1_secret': self.secret}])
+
+    def test_secret_branch_error_same_branch(self):
+        # Test that we are unable to define a secret twice on the same
+        # project-branch.
+        in_repo_conf = textwrap.dedent(
+            """
+            - secret:
+                name: project1_secret
+                data: {}
+            - secret:
+                name: project1_secret
+                data: {}
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('already defined', A.messages[0])
+
+    def test_secret_branch_error_same_project(self):
+        # Test that we are unable to create a secret which differs
+        # from another with the same name -- i.e., that if we have a
+        # duplicate secret on multiple branches of the same project,
+        # they must be identical.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - secret:
+                name: project1_secret
+                data: {}
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('does not match existing definition in branch master',
+                      A.messages[0])
+
+    def test_secret_branch_error_other_project(self):
+        # Test that we are unable to create a secret with the same
+        # name as another.  We're never allowed to have a secret with
+        # the same name outside of a project.
+        in_repo_conf = textwrap.dedent(
+            """
+            - secret:
+                name: project1_secret
+                data: {}
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('already defined in project org/project1',
+                      A.messages[0])
+
+
 class TestSecretInheritance(ZuulTestCase):
     tenant_config_file = 'config/secret-inheritance/main.yaml'
 
@@ -2724,6 +2941,278 @@
         self._test_secret_file_fail()
 
 
+class TestNodesets(ZuulTestCase):
+    tenant_config_file = 'config/nodesets/main.yaml'
+
+    def test_nodeset_branch(self):
+        # Test that we can use a nodeset defined in another branch of
+        # the same project.
+        self.create_branch('org/project2', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project2', 'stable'))
+        self.waitUntilSettled()
+
+        with open(os.path.join(FIXTURE_DIR,
+                               'config/nodesets/git/',
+                               'org_project2/zuul-nodeset.yaml')) as f:
+            config = f.read()
+
+        file_dict = {'zuul.yaml': config}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                parent: base
+                name: project2-test
+                nodeset: project2-nodeset
+
+            - project:
+                check:
+                  jobs:
+                    - project2-test
+                gate:
+                  jobs:
+                    - noop
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        B = self.fake_gerrit.addFakeChange('org/project2', 'stable', 'B',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(B.reported, 1, "B should report success")
+        self.assertHistory([
+            dict(name='project2-test', result='SUCCESS', changes='2,1',
+                 node='ubuntu-xenial'),
+        ])
+
+    def test_nodeset_branch_duplicate(self):
+        # Test that we can create a duplicate nodeset on a different
+        # branch of the same project -- i.e., that when we branch
+        # master to stable on a project with a nodeset, nothing
+        # changes.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+        self.waitUntilSettled()
+
+        A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 1,
+                         "A should report success")
+        self.assertHistory([
+            dict(name='project1-test', result='SUCCESS', changes='1,1',
+                 node='ubuntu-xenial'),
+        ])
+
+    def test_nodeset_branch_error_same_branch(self):
+        # Test that we are unable to define a nodeset twice on the same
+        # project-branch.
+        in_repo_conf = textwrap.dedent(
+            """
+            - nodeset:
+                name: project1-nodeset
+                nodes: []
+            - nodeset:
+                name: project1-nodeset
+                nodes: []
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('already defined', A.messages[0])
+
+    def test_nodeset_branch_error_same_project(self):
+        # Test that we are unable to create a nodeset which differs
+        # from another with the same name -- i.e., that if we have a
+        # duplicate nodeset on multiple branches of the same project,
+        # they must be identical.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - nodeset:
+                name: project1-nodeset
+                nodes: []
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('does not match existing definition in branch master',
+                      A.messages[0])
+
+    def test_nodeset_branch_error_other_project(self):
+        # Test that we are unable to create a nodeset with the same
+        # name as another.  We're never allowed to have a nodeset with
+        # the same name outside of a project.
+        in_repo_conf = textwrap.dedent(
+            """
+            - nodeset:
+                name: project1-nodeset
+                nodes: []
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('already defined in project org/project1',
+                      A.messages[0])
+
+
+class TestSemaphoreBranches(ZuulTestCase):
+    tenant_config_file = 'config/semaphore-branches/main.yaml'
+
+    def test_semaphore_branch(self):
+        # Test that we can use a semaphore defined in another branch of
+        # the same project.
+        self.create_branch('org/project2', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project2', 'stable'))
+        self.waitUntilSettled()
+
+        with open(os.path.join(FIXTURE_DIR,
+                               'config/semaphore-branches/git/',
+                               'org_project2/zuul-semaphore.yaml')) as f:
+            config = f.read()
+
+        file_dict = {'zuul.yaml': config}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                parent: base
+                name: project2-test
+                semaphore: project2-semaphore
+
+            - project:
+                check:
+                  jobs:
+                    - project2-test
+                gate:
+                  jobs:
+                    - noop
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        B = self.fake_gerrit.addFakeChange('org/project2', 'stable', 'B',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(B.reported, 1, "B should report success")
+        self.assertHistory([
+            dict(name='project2-test', result='SUCCESS', changes='2,1')
+        ])
+
+    def test_semaphore_branch_duplicate(self):
+        # Test that we can create a duplicate semaphore on a different
+        # branch of the same project -- i.e., that when we branch
+        # master to stable on a project with a semaphore, nothing
+        # changes.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+        self.waitUntilSettled()
+
+        A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 1,
+                         "A should report success")
+        self.assertHistory([
+            dict(name='project1-test', result='SUCCESS', changes='1,1')
+        ])
+
+    def test_semaphore_branch_error_same_branch(self):
+        # Test that we are unable to define a semaphore twice on the same
+        # project-branch.
+        in_repo_conf = textwrap.dedent(
+            """
+            - semaphore:
+                name: project1-semaphore
+                max: 2
+            - semaphore:
+                name: project1-semaphore
+                max: 2
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('already defined', A.messages[0])
+
+    def test_semaphore_branch_error_same_project(self):
+        # Test that we are unable to create a semaphore which differs
+        # from another with the same name -- i.e., that if we have a
+        # duplicate semaphore on multiple branches of the same project,
+        # they must be identical.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - semaphore:
+                name: project1-semaphore
+                max: 4
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('does not match existing definition in branch master',
+                      A.messages[0])
+
+    def test_semaphore_branch_error_other_project(self):
+        # Test that we are unable to create a semaphore with the same
+        # name as another.  We're never allowed to have a semaphore with
+        # the same name outside of a project.
+        in_repo_conf = textwrap.dedent(
+            """
+            - semaphore:
+                name: project1-semaphore
+                max: 2
+            """)
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('already defined in project org/project1',
+                      A.messages[0])
+
+
 class TestJobOutput(AnsibleZuulTestCase):
     tenant_config_file = 'config/job-output/main.yaml'
 
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 6881a83..b5ebe9f 100644
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -78,14 +78,105 @@
         super(TestWeb, self).tearDown()
 
     def test_web_status(self):
-        "Test that we can filter to only certain changes in the webapp."
+        "Test that we can retrieve JSON status info"
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+
+        self.executor_server.release('project-merge')
+        self.waitUntilSettled()
 
         req = urllib.request.Request(
             "http://localhost:%s/tenant-one/status.json" % self.port)
         f = urllib.request.urlopen(req)
-        data = json.loads(f.read().decode('utf8'))
+        headers = f.info()
+        self.assertIn('Content-Length', headers)
+        self.assertIn('Content-Type', headers)
+        self.assertEqual(
+            'application/json; charset=utf-8', headers['Content-Type'])
+        self.assertIn('Access-Control-Allow-Origin', headers)
+        self.assertIn('Cache-Control', headers)
+        self.assertIn('Last-Modified', headers)
+        data = f.read().decode('utf8')
 
-        self.assertIn('pipelines', data)
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        data = json.loads(data)
+        status_jobs = []
+        for p in data['pipelines']:
+            for q in p['change_queues']:
+                if p['name'] in ['gate', 'conflict']:
+                    self.assertEqual(q['window'], 20)
+                else:
+                    self.assertEqual(q['window'], 0)
+                for head in q['heads']:
+                    for change in head:
+                        self.assertTrue(change['active'])
+                        self.assertIn(change['id'], ('1,1', '2,1', '3,1'))
+                        for job in change['jobs']:
+                            status_jobs.append(job)
+        self.assertEqual('project-merge', status_jobs[0]['name'])
+        # TODO(mordred) pull uuids from self.builds
+        self.assertEqual(
+            'stream.html?uuid={uuid}&logfile=console.log'.format(
+                uuid=status_jobs[0]['uuid']),
+            status_jobs[0]['url'])
+        self.assertEqual(
+            'finger://{hostname}/{uuid}'.format(
+                hostname=self.executor_server.hostname,
+                uuid=status_jobs[0]['uuid']),
+            status_jobs[0]['finger_url'])
+        # TOOD(mordred) configure a success-url on the base job
+        self.assertEqual(
+            'finger://{hostname}/{uuid}'.format(
+                hostname=self.executor_server.hostname,
+                uuid=status_jobs[0]['uuid']),
+            status_jobs[0]['report_url'])
+        self.assertEqual('project-test1', status_jobs[1]['name'])
+        self.assertEqual(
+            'stream.html?uuid={uuid}&logfile=console.log'.format(
+                uuid=status_jobs[1]['uuid']),
+            status_jobs[1]['url'])
+        self.assertEqual(
+            'finger://{hostname}/{uuid}'.format(
+                hostname=self.executor_server.hostname,
+                uuid=status_jobs[1]['uuid']),
+            status_jobs[1]['finger_url'])
+        self.assertEqual(
+            'finger://{hostname}/{uuid}'.format(
+                hostname=self.executor_server.hostname,
+                uuid=status_jobs[1]['uuid']),
+            status_jobs[1]['report_url'])
+
+        self.assertEqual('project-test2', status_jobs[2]['name'])
+        self.assertEqual(
+            'stream.html?uuid={uuid}&logfile=console.log'.format(
+                uuid=status_jobs[2]['uuid']),
+            status_jobs[2]['url'])
+        self.assertEqual(
+            'finger://{hostname}/{uuid}'.format(
+                hostname=self.executor_server.hostname,
+                uuid=status_jobs[2]['uuid']),
+            status_jobs[2]['finger_url'])
+        self.assertEqual(
+            'finger://{hostname}/{uuid}'.format(
+                hostname=self.executor_server.hostname,
+                uuid=status_jobs[2]['uuid']),
+            status_jobs[2]['report_url'])
+
+        # check job dependencies
+        self.assertIsNotNone(status_jobs[0]['dependencies'])
+        self.assertIsNotNone(status_jobs[1]['dependencies'])
+        self.assertIsNotNone(status_jobs[2]['dependencies'])
+        self.assertEqual(len(status_jobs[0]['dependencies']), 0)
+        self.assertEqual(len(status_jobs[1]['dependencies']), 1)
+        self.assertEqual(len(status_jobs[2]['dependencies']), 1)
+        self.assertIn('project-merge', status_jobs[1]['dependencies'])
+        self.assertIn('project-merge', status_jobs[2]['dependencies'])
 
     def test_web_bad_url(self):
         # do we 404 correctly
diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py
deleted file mode 100644
index c06fc93..0000000
--- a/tests/unit/test_webapp.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2014 Hewlett-Packard Development Company, L.P.
-# Copyright 2014 Rackspace Australia
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os
-import json
-import urllib
-
-import webob
-
-from tests.base import ZuulTestCase, FIXTURE_DIR
-
-
-class TestWebapp(ZuulTestCase):
-    tenant_config_file = 'config/single-tenant/main.yaml'
-
-    def setUp(self):
-        super(TestWebapp, self).setUp()
-        self.executor_server.hold_jobs_in_build = True
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        A.addApproval('Code-Review', 2)
-        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
-        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
-        B.addApproval('Code-Review', 2)
-        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
-        self.waitUntilSettled()
-        self.port = self.webapp.server.socket.getsockname()[1]
-
-    def tearDown(self):
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-        self.waitUntilSettled()
-        super(TestWebapp, self).tearDown()
-
-    def test_webapp_status(self):
-        "Test that we can filter to only certain changes in the webapp."
-
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/status" % self.port)
-        f = urllib.request.urlopen(req)
-        data = json.loads(f.read().decode('utf8'))
-
-        self.assertIn('pipelines', data)
-
-    def test_webapp_status_compat(self):
-        # testing compat with status.json
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/status.json" % self.port)
-        f = urllib.request.urlopen(req)
-        data = json.loads(f.read().decode('utf8'))
-
-        self.assertIn('pipelines', data)
-
-    def test_webapp_bad_url(self):
-        # do we 404 correctly
-        req = urllib.request.Request(
-            "http://localhost:%s/status/foo" % self.port)
-        self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, req)
-
-    def test_webapp_find_change(self):
-        # can we filter by change id
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/status/change/1,1" % self.port)
-        f = urllib.request.urlopen(req)
-        data = json.loads(f.read().decode('utf8'))
-
-        self.assertEqual(1, len(data), data)
-        self.assertEqual("org/project", data[0]['project'])
-
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/status/change/2,1" % self.port)
-        f = urllib.request.urlopen(req)
-        data = json.loads(f.read().decode('utf8'))
-
-        self.assertEqual(1, len(data), data)
-        self.assertEqual("org/project1", data[0]['project'], data)
-
-    def test_webapp_keys(self):
-        with open(os.path.join(FIXTURE_DIR, 'public.pem'), 'rb') as f:
-            public_pem = f.read()
-
-        req = urllib.request.Request(
-            "http://localhost:%s/tenant-one/keys/gerrit/org/project.pub" %
-            self.port)
-        f = urllib.request.urlopen(req)
-        self.assertEqual(f.read(), public_pem)
-
-    def test_webapp_custom_handler(self):
-        def custom_handler(path, tenant_name, request):
-            return webob.Response(body='ok')
-
-        self.webapp.register_path('/custom', custom_handler)
-        req = urllib.request.Request(
-            "http://localhost:%s/custom" % self.port)
-        f = urllib.request.urlopen(req)
-        self.assertEqual(b'ok', f.read())
-
-        self.webapp.unregister_path('/custom')
-        self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, req)
-
-    def test_webapp_404_on_unknown_tenant(self):
-        req = urllib.request.Request(
-            "http://localhost:{}/non-tenant/status.json".format(self.port))
-        e = self.assertRaises(
-            urllib.error.HTTPError, urllib.request.urlopen, req)
-        self.assertEqual(404, e.code)
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
index 4cb1666..45ad68c 100755
--- a/tools/encrypt_secret.py
+++ b/tools/encrypt_secret.py
@@ -26,9 +26,11 @@
 try:
     from urllib.request import Request
     from urllib.request import urlopen
+    from urllib.parse import urlparse
 except ImportError:
     from urllib2 import Request
     from urllib2 import urlopen
+    from urlparse import urlparse
 
 DESCRIPTION = """Encrypt a secret for Zuul.
 
@@ -43,7 +45,6 @@
     parser.add_argument('url',
                         help="The base URL of the zuul server and tenant.  "
                         "E.g., https://zuul.example.com/tenant-name")
-    # TODO(jeblair): Throw a fit if SSL is not used.
     parser.add_argument('project',
                         help="The name of the project.")
     parser.add_argument('--strip', action='store_true', default=False,
@@ -60,6 +61,15 @@
                         "to standard output.")
     args = parser.parse_args()
 
+    # We should not use unencrypted connections for retrieving the public key.
+    # Otherwise our secret can be compromised. The schemes file and https are
+    # considered safe.
+    url = urlparse(args.url)
+    if url.scheme not in ('file', 'https'):
+        sys.stderr.write("WARNING: Retrieving encryption key via an "
+                         "unencrypted connection. Your secret may get "
+                         "compromised.\n")
+
     req = Request("%s/%s.pub" % (args.url.rstrip('/'), args.project))
     pubkey = urlopen(req)
 
diff --git a/tools/github-debugging.py b/tools/github-debugging.py
index 101fd11..da6fd0c 100755
--- a/tools/github-debugging.py
+++ b/tools/github-debugging.py
@@ -11,6 +11,8 @@
 # TODO: for real use override the following variables
 server = 'github.com'
 api_token = 'xxxx'
+appid = 2
+appkey = '/opt/project/appkey'
 
 org = 'example'
 repo = 'sandbox'
@@ -42,20 +44,36 @@
     return conn
 
 
+def create_connection_app(server, appid, appkey):
+    driver = GithubDriver()
+    connection_config = {
+        'server': server,
+        'app_id': appid,
+        'app_key': appkey,
+    }
+    conn = GithubConnection(driver, 'github', connection_config)
+    conn._authenticateGithubAPI()
+    conn._prime_installation_map()
+    return conn
+
+
 def get_change(connection: GithubConnection,
                org: str,
                repo: str,
                pull: int) -> Change:
     p = Project("%s/%s" % (org, repo), connection.source)
-    github = connection.getGithubClient(p)
+    github = connection.getGithubClient(p.name)
     pr = github.pull_request(org, repo, pull)
     sha = pr.head.sha
     return conn._getChange(p, pull, sha, True)
 
 
-# create github connection
+# create github connection with api token
 conn = create_connection(server, api_token)
 
+# create github connection with app key
+# conn = create_connection_app(server, appid, appkey)
+
 
 # Now we can do anything we want with the connection, e.g. check canMerge for
 # a pull request.
diff --git a/tools/nodepool-integration-setup.sh b/tools/nodepool-integration-setup.sh
index c02a016..58c39cf 100755
--- a/tools/nodepool-integration-setup.sh
+++ b/tools/nodepool-integration-setup.sh
@@ -3,7 +3,7 @@
 /usr/zuul-env/bin/zuul-cloner --workspace /tmp --cache-dir /opt/git \
     git://git.openstack.org openstack-infra/nodepool
 
-ln -s /tmp/nodepool/log $WORKSPACE/logs
+ln -s /tmp/nodepool/log $HOME/logs
 
 cd /tmp/openstack-infra/nodepool
 /usr/local/jenkins/slave_scripts/install-distro-packages.sh
diff --git a/tox.ini b/tox.ini
index 5efc4c0..e5035bd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,7 +6,7 @@
 [testenv]
 basepython = python3
 setenv = VIRTUAL_ENV={envdir}
-         OS_TEST_TIMEOUT=120
+         OS_TEST_TIMEOUT=150
 passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS
 usedevelop = True
 install_command = pip install {opts} {packages}
@@ -36,7 +36,8 @@
   python setup.py test --coverage
 
 [testenv:docs]
-commands = python setup.py build_sphinx
+commands =
+  sphinx-build -W -d doc/build/doctrees -b html doc/source/ doc/build/html
 
 [testenv:venv]
 commands = {posargs}
@@ -49,6 +50,6 @@
 [flake8]
 # These are ignored intentionally in openstack-infra projects;
 # please don't submit patches that solely correct them or enable them.
-ignore = E125,E129,E402,E741,H,W503
+ignore = E124,E125,E129,E402,E741,H,W503
 show-source = True
 exclude = .venv,.tox,dist,doc,build,*.egg
diff --git a/zuul/ansible/action/normal.py b/zuul/ansible/action/normal.py
index 152f13f..35ae8cb 100644
--- a/zuul/ansible/action/normal.py
+++ b/zuul/ansible/action/normal.py
@@ -63,6 +63,8 @@
 
         Block any access of files outside the zuul work dir.
         '''
+        if self._task.args.get('get_mime') is not None:
+            raise AnsibleError("get_mime on localhost is forbidden")
         paths._fail_if_unsafe(self._task.args['path'])
 
     def handle_file(self):
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index df28a57..15b491c 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -367,12 +367,13 @@
                 self._log_message(
                     result, status='MODULE FAILURE',
                     msg=result_dict['module_stderr'])
-        elif (len([key for key in result_dict.keys()
-                   if not key.startswith('_ansible')]) == 1):
+        elif result._task.action == 'debug':
             # this is a debug statement, handle it special
             for key in [k for k in result_dict.keys()
                         if k.startswith('_ansible')]:
                 del result_dict[key]
+            if 'changed' in result_dict.keys():
+                del result_dict['changed']
             keyname = next(iter(result_dict.keys()))
             # If it has msg, that means it was like:
             #
diff --git a/zuul/ansible/library/command.py b/zuul/ansible/library/command.py
old mode 100644
new mode 100755
index 0fc6129..9be3e2f
--- a/zuul/ansible/library/command.py
+++ b/zuul/ansible/library/command.py
@@ -371,9 +371,12 @@
 
         # ZUUL: Replaced the excution loop with the zuul_runner run function
         cmd = subprocess.Popen(args, **kwargs)
-        t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id))
-        t.daemon = True
-        t.start()
+        if self.no_log:
+            t = None
+        else:
+            t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id))
+            t.daemon = True
+            t.start()
 
         ret = cmd.wait()
 
@@ -381,22 +384,25 @@
         # to catch up and exit.  If it hasn't done so by then, it is very
         # likely stuck in readline() because it spawed a child that is
         # holding stdout or stderr open.
-        t.join(10)
-        with Console(zuul_log_id) as console:
-            if t.isAlive():
-                console.addLine("[Zuul] standard output/error still open "
-                                "after child exited")
-            console.addLine("[Zuul] Task exit code: %s\n" % ret)
+        if t:
+            t.join(10)
+            with Console(zuul_log_id) as console:
+                if t.isAlive():
+                    console.addLine("[Zuul] standard output/error still open "
+                                    "after child exited")
+                console.addLine("[Zuul] Task exit code: %s\n" % ret)
 
-        # ZUUL: If the console log follow thread *is* stuck in readline,
-        # we can't close stdout (attempting to do so raises an
-        # exception) , so this is disabled.
-        # cmd.stdout.close()
-        # cmd.stderr.close()
+            # ZUUL: If the console log follow thread *is* stuck in readline,
+            # we can't close stdout (attempting to do so raises an
+            # exception) , so this is disabled.
+            # cmd.stdout.close()
+            # cmd.stderr.close()
 
-        # ZUUL: stdout and stderr are in the console log file
-        # ZUUL: return the saved log lines so we can ship them back
-        stdout = b('').join(_log_lines)
+            # ZUUL: stdout and stderr are in the console log file
+            # ZUUL: return the saved log lines so we can ship them back
+            stdout = b('').join(_log_lines)
+        else:
+            stdout = b('')
         stderr = b('')
 
         rc = cmd.returncode
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index bf11c6f..b299219 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -171,14 +171,6 @@
         return pid_fn
 
     @abc.abstractmethod
-    def exit_handler(self, signum, frame):
-        """
-        This is a signal handler which is called on SIGINT and SIGTERM and must
-        take care of stopping the application.
-        """
-        pass
-
-    @abc.abstractmethod
     def run(self):
         """
         This is the main run method of the application.
@@ -197,8 +189,6 @@
         signal.signal(signal.SIGUSR2, stack_dump_handler)
 
         if self.args.nodaemon:
-            signal.signal(signal.SIGTERM, self.exit_handler)
-            signal.signal(signal.SIGINT, self.exit_handler)
             self.run()
         else:
             # Exercise the pidfile before we do anything else (including
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index ebf59b9..a7b3ef3 100755
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -51,6 +51,11 @@
                                   required=True)
         cmd_autohold.add_argument('--job', help='job name',
                                   required=True)
+        cmd_autohold.add_argument('--change',
+                                  help='specific change to hold nodes for',
+                                  required=False, default='')
+        cmd_autohold.add_argument('--ref', help='git ref to hold nodes for',
+                                  required=False, default='')
         cmd_autohold.add_argument('--reason', help='reason for the hold',
                                   required=True)
         cmd_autohold.add_argument('--count',
@@ -173,9 +178,15 @@
     def autohold(self):
         client = zuul.rpcclient.RPCClient(
             self.server, self.port, self.ssl_key, self.ssl_cert, self.ssl_ca)
+        if self.args.change and self.args.ref:
+            print("Change and ref can't be both used for the same request")
+            return False
+
         r = client.autohold(tenant=self.args.tenant,
                             project=self.args.project,
                             job=self.args.job,
+                            change=self.args.change,
+                            ref=self.args.ref,
                             reason=self.args.reason,
                             count=self.args.count)
         return r
@@ -190,14 +201,19 @@
             return True
 
         table = prettytable.PrettyTable(
-            field_names=['Tenant', 'Project', 'Job', 'Count', 'Reason'])
+            field_names=[
+                'Tenant', 'Project', 'Job', 'Ref Filter', 'Count', 'Reason'
+            ])
 
         for key, value in autohold_requests.items():
             # The key comes to us as a CSV string because json doesn't like
             # non-str keys.
-            tenant_name, project_name, job_name = key.split(',')
+            tenant_name, project_name, job_name, ref_filter = key.split(',')
             count, reason = value
-            table.add_row([tenant_name, project_name, job_name, count, reason])
+
+            table.add_row([
+                tenant_name, project_name, job_name, ref_filter, count, reason
+            ])
         print(table)
         return True
 
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 5b06f0c..b050a59 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -17,6 +17,7 @@
 import logging
 import os
 import sys
+import signal
 import tempfile
 
 import zuul.cmd
@@ -50,6 +51,8 @@
 
     def exit_handler(self, signum, frame):
         self.executor.stop()
+        self.executor.join()
+        sys.exit(0)
 
     def start_log_streamer(self):
         pipe_read, pipe_write = os.pipe()
@@ -106,7 +109,16 @@
                                        log_streaming_port=self.finger_port)
         self.executor.start()
 
-        self.executor.join()
+        if self.args.nodaemon:
+            signal.signal(signal.SIGTERM, self.exit_handler)
+            while True:
+                try:
+                    signal.pause()
+                except KeyboardInterrupt:
+                    print("Ctrl + C: asking executor to exit nicely...\n")
+                    self.exit_handler(signal.SIGINT, None)
+        else:
+            self.executor.join()
 
 
 def main():
diff --git a/zuul/cmd/fingergw.py b/zuul/cmd/fingergw.py
index 0d47f08..920eed8 100644
--- a/zuul/cmd/fingergw.py
+++ b/zuul/cmd/fingergw.py
@@ -14,6 +14,7 @@
 # under the License.
 
 import logging
+import signal
 import sys
 
 import zuul.cmd
@@ -46,9 +47,6 @@
         if self.args.command:
             self.args.nodaemon = True
 
-    def exit_handler(self, signum, frame):
-        self.stop()
-
     def run(self):
         '''
         Main entry point for the FingerGatewayApp.
@@ -86,7 +84,19 @@
         self.log.info('Starting Zuul finger gateway app')
         self.gateway.start()
 
-        self.gateway.wait()
+        if self.args.nodaemon:
+            # NOTE(Shrews): When running in non-daemon mode, although sending
+            # the 'stop' command via the command socket will shutdown the
+            # gateway, it's still necessary to Ctrl+C to stop the app.
+            while True:
+                try:
+                    signal.pause()
+                except KeyboardInterrupt:
+                    print("Ctrl + C: asking gateway to exit nicely...\n")
+                    self.stop()
+                    break
+        else:
+            self.gateway.wait()
 
         self.log.info('Stopped Zuul finger gateway app')
 
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
index 390191f..8c47989 100755
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -14,6 +14,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import signal
 import sys
 
 import zuul.cmd
@@ -43,6 +44,8 @@
 
     def exit_handler(self, signum, frame):
         self.merger.stop()
+        self.merger.join()
+        sys.exit(0)
 
     def run(self):
         # See comment at top of file about zuul imports
@@ -59,7 +62,16 @@
                                                      self.connections)
         self.merger.start()
 
-        self.merger.join()
+        if self.args.nodaemon:
+            signal.signal(signal.SIGTERM, self.exit_handler)
+            while True:
+                try:
+                    signal.pause()
+                except KeyboardInterrupt:
+                    print("Ctrl + C: asking merger to exit nicely...\n")
+                    self.exit_handler(signal.SIGINT, None)
+        else:
+            self.merger.join()
 
 
 def main():
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index 3cffa10..68c9000 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -63,7 +63,9 @@
 
     def exit_handler(self, signum, frame):
         self.sched.exit()
+        self.sched.join()
         self.stop_gear_server()
+        sys.exit(0)
 
     def start_gear_server(self):
         pipe_read, pipe_write = os.pipe()
@@ -118,7 +120,6 @@
         import zuul.executor.client
         import zuul.merger.client
         import zuul.nodepool
-        import zuul.webapp
         import zuul.zk
 
         if (self.config.has_option('gearman_server', 'start') and
@@ -142,15 +143,6 @@
 
         zookeeper.connect(zookeeper_hosts, timeout=zookeeper_timeout)
 
-        cache_expiry = get_default(self.config, 'webapp', 'status_expiry', 1)
-        listen_address = get_default(self.config, 'webapp', 'listen_address',
-                                     '0.0.0.0')
-        port = get_default(self.config, 'webapp', 'port', 8001)
-
-        webapp = zuul.webapp.WebApp(
-            self.sched, port=port, cache_expiry=cache_expiry,
-            listen_address=listen_address)
-
         self.configure_connections()
         self.sched.setExecutor(gearman)
         self.sched.setMerger(merger)
@@ -160,7 +152,7 @@
         self.log.info('Starting scheduler')
         try:
             self.sched.start()
-            self.sched.registerConnections(self.connections, webapp)
+            self.sched.registerConnections(self.connections)
             self.sched.reconfigure(self.config)
             self.sched.resume()
         except Exception:
@@ -168,12 +160,19 @@
             # TODO(jeblair): If we had all threads marked as daemon,
             # we might be able to have a nicer way of exiting here.
             sys.exit(1)
-        self.log.info('Starting Webapp')
-        webapp.start()
 
         signal.signal(signal.SIGHUP, self.reconfigure_handler)
 
-        self.sched.join()
+        if self.args.nodaemon:
+            signal.signal(signal.SIGTERM, self.exit_handler)
+            while True:
+                try:
+                    signal.pause()
+                except KeyboardInterrupt:
+                    print("Ctrl + C: asking scheduler to exit nicely...\n")
+                    self.exit_handler(signal.SIGINT, None)
+        else:
+            self.sched.join()
 
 
 def main():
diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py
index 78392db..abdb1cb 100755
--- a/zuul/cmd/web.py
+++ b/zuul/cmd/web.py
@@ -22,7 +22,6 @@
 import zuul.cmd
 import zuul.web
 
-from zuul.driver.sql import sqlconnection
 from zuul.lib.config import get_default
 
 
@@ -49,29 +48,15 @@
         params['ssl_cert'] = get_default(self.config, 'gearman', 'ssl_cert')
         params['ssl_ca'] = get_default(self.config, 'gearman', 'ssl_ca')
 
-        sql_conn_name = get_default(self.config, 'web',
-                                    'sql_connection_name')
-        sql_conn = None
-        if sql_conn_name:
-            # we want a specific sql connection
-            sql_conn = self.connections.connections.get(sql_conn_name)
-            if not sql_conn:
-                self.log.error("Couldn't find sql connection '%s'" %
-                               sql_conn_name)
+        params['connections'] = []
+        # Validate config here before we spin up the ZuulWeb object
+        for conn_name, connection in self.connections.connections.items():
+            try:
+                if connection.validateWebConfig(self.config, self.connections):
+                    params['connections'].append(connection)
+            except Exception:
+                self.log.exception("Error validating config")
                 sys.exit(1)
-        else:
-            # look for any sql connection
-            connections = [c for c in self.connections.connections.values()
-                           if isinstance(c, sqlconnection.SQLConnection)]
-            if len(connections) > 1:
-                self.log.error("Multiple sql connection found, "
-                               "set the sql_connection_name option "
-                               "in zuul.conf [web] section")
-                sys.exit(1)
-            if connections:
-                # use this sql connection by default
-                sql_conn = connections[0]
-        params['sql_connection'] = sql_conn
 
         try:
             self.web = zuul.web.ZuulWeb(**params)
@@ -89,6 +74,12 @@
                                        name='web')
         self.thread.start()
 
+        try:
+            signal.pause()
+        except KeyboardInterrupt:
+            print("Ctrl + C: asking web server to exit nicely...\n")
+            self.exit_handler(signal.SIGINT, None)
+
         self.thread.join()
         loop.stop()
         loop.close()
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d622370..be1bd63 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -407,7 +407,7 @@
     @staticmethod
     def fromYaml(conf, anonymous=False):
         NodeSetParser.getSchema(anonymous)(conf)
-        ns = model.NodeSet(conf.get('name'))
+        ns = model.NodeSet(conf.get('name'), conf.get('_source_context'))
         node_names = set()
         group_names = set()
         for conf_node in as_list(conf['nodes']):
@@ -700,10 +700,10 @@
                 (trusted, project) = tenant.getProject(project_name)
                 if project is None:
                     raise Exception("Unknown project %s" % (project_name,))
-                job_project = model.JobProject(project_name,
+                job_project = model.JobProject(project.canonical_name,
                                                project_override_branch,
                                                project_override_checkout)
-                new_projects[project_name] = job_project
+                new_projects[project.canonical_name] = job_project
             job.required_projects = new_projects
 
         tags = conf.get('tags')
@@ -1597,7 +1597,8 @@
             classes = TenantParser._getLoadClasses(tenant, config_secret)
             if 'secret' not in classes:
                 continue
-            layout.addSecret(SecretParser.fromYaml(layout, config_secret))
+            with configuration_exceptions('secret', config_secret):
+                layout.addSecret(SecretParser.fromYaml(layout, config_secret))
 
         for config_job in data.jobs:
             classes = TenantParser._getLoadClasses(tenant, config_job)
@@ -1621,23 +1622,22 @@
                 if parent:
                     layout.getJob(parent)
 
-        if not skip_semaphores:
-            for config_semaphore in data.semaphores:
-                classes = TenantParser._getLoadClasses(
-                    tenant, config_semaphore)
-                if 'semaphore' not in classes:
-                    continue
+        if skip_semaphores:
+            # We should not actually update the layout with new
+            # semaphores, but so that we can validate that the config
+            # is correct, create a shadow layout here to which we add
+            # new semaphores so validation is complete.
+            semaphore_layout = model.Layout(tenant)
+        else:
+            semaphore_layout = layout
+        for config_semaphore in data.semaphores:
+            classes = TenantParser._getLoadClasses(
+                tenant, config_semaphore)
+            if 'semaphore' not in classes:
+                continue
+            with configuration_exceptions('semaphore', config_semaphore):
                 semaphore = SemaphoreParser.fromYaml(config_semaphore)
-                old_semaphore = layout.semaphores.get(semaphore.name)
-                if (old_semaphore and
-                    (old_semaphore.source_context.project ==
-                     semaphore.source_context.project)):
-                    # If a semaphore shows up twice in the same
-                    # project, it's probably due to showing up in
-                    # two branches.  Ignore subsequent
-                    # definitions.
-                    continue
-                layout.addSemaphore(semaphore)
+                semaphore_layout.addSemaphore(semaphore)
 
         project_template_parser = ProjectTemplateParser(tenant, layout)
         for config_template in data.project_templates:
diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py
index 483495d..86f14d6 100644
--- a/zuul/connection/__init__.py
+++ b/zuul/connection/__init__.py
@@ -75,20 +75,29 @@
         still in use.  Anything in our cache that isn't in the supplied
         list should be safe to remove from the cache."""
 
-    def registerWebapp(self, webapp):
-        self.webapp = webapp
+    def getWebHandlers(self, zuul_web):
+        """Return a list of web handlers to register with zuul-web.
 
-    def registerHttpHandler(self, path, handler):
-        """Add connection handler for HTTP URI.
-
-        Connection can use builtin HTTP server for listening on incoming event
-        requests. The resulting path will be /connection/connection_name/path.
+        :param zuul.web.ZuulWeb zuul_web:
+            Zuul Web instance.
+        :returns: List of `zuul.web.handler.BaseWebHandler` instances.
         """
-        self.webapp.register_path(self._connectionPath(path), handler)
+        return []
 
-    def unregisterHttpHandler(self, path):
-        """Remove the connection handler for HTTP URI."""
-        self.webapp.unregister_path(self._connectionPath(path))
+    def validateWebConfig(self, config, connections):
+        """Validate config and determine whether to register web handlers.
 
-    def _connectionPath(self, path):
-        return '/connection/%s/%s' % (self.connection_name, path)
+        By default this method returns False, which means this connection
+        has no web handlers to register.
+
+        If the method returns True, then its `getWebHandlers` method
+        should be called during route registration.
+
+        If there is a fatal error, the method should raise an exception.
+
+        :param config:
+           The parsed config object.
+        :param zuul.lib.connections.ConnectionRegistry connections:
+           Registry of all configured connections.
+        """
+        return False
diff --git a/zuul/driver/gerrit/gerritsource.py b/zuul/driver/gerrit/gerritsource.py
index 9e327b9..8f3408e 100644
--- a/zuul/driver/gerrit/gerritsource.py
+++ b/zuul/driver/gerrit/gerritsource.py
@@ -54,7 +54,10 @@
             parsed = urllib.parse.urlparse(url)
         except ValueError:
             return None
-        m = self.change_re.match(parsed.path)
+        path = parsed.path
+        if parsed.fragment:
+            path += '#' + parsed.fragment
+        m = self.change_re.match(path)
         if not m:
             return None
         try:
@@ -138,6 +141,10 @@
         )
         return [f]
 
+    def getRefForChange(self, change):
+        partial = change[-2:]
+        return "refs/changes/%s/%s/.*" % (partial, change)
+
 
 approval = vs.Schema({'username': str,
                       'email': str,
diff --git a/zuul/driver/git/gitsource.py b/zuul/driver/git/gitsource.py
index a7d42be..9f0963d 100644
--- a/zuul/driver/git/gitsource.py
+++ b/zuul/driver/git/gitsource.py
@@ -68,3 +68,6 @@
 
     def getRejectFilters(self, config):
         return []
+
+    def getRefForChange(self, change):
+        raise NotImplemented()
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index b766c6f..6dfcdd3 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -21,20 +21,25 @@
 import threading
 import time
 import re
+import json
+import traceback
 
+from aiohttp import web
 import cachecontrol
 from cachecontrol.cache import DictCache
 from cachecontrol.heuristics import BaseHeuristic
 import iso8601
 import jwt
 import requests
-import webob
-import webob.dec
 import voluptuous as v
 import github3
 import github3.exceptions
 
+import gear
+
 from zuul.connection import BaseConnection
+from zuul.web.handler import BaseDriverWebHandler
+from zuul.lib.config import get_default
 from zuul.model import Ref, Branch, Tag, Project
 from zuul.exceptions import MergeFailure
 from zuul.driver.github.githubmodel import PullRequest, GithubTriggerEvent
@@ -65,71 +70,101 @@
 utc = UTC()
 
 
-class GithubWebhookListener():
-
-    log = logging.getLogger("zuul.GithubWebhookListener")
+class GithubGearmanWorker(object):
+    """A thread that answers gearman requests"""
+    log = logging.getLogger("zuul.GithubGearmanWorker")
 
     def __init__(self, connection):
+        self.config = connection.sched.config
         self.connection = connection
+        self.thread = threading.Thread(target=self._run,
+                                       name='github-gearman-worker')
+        self._running = False
+        handler = "github:%s:payload" % self.connection.connection_name
+        self.jobs = {
+            handler: self.handle_payload,
+        }
 
-    def handle_request(self, path, tenant_name, request):
-        if request.method != 'POST':
-            self.log.debug("Only POST method is allowed.")
-            raise webob.exc.HTTPMethodNotAllowed(
-                'Only POST method is allowed.')
+    def _run(self):
+        while self._running:
+            try:
+                job = self.gearman.getJob()
+                try:
+                    if job.name not in self.jobs:
+                        self.log.exception("Exception while running job")
+                        job.sendWorkException(
+                            traceback.format_exc().encode('utf8'))
+                        continue
+                    output = self.jobs[job.name](json.loads(job.arguments))
+                    job.sendWorkComplete(json.dumps(output))
+                except Exception:
+                    self.log.exception("Exception while running job")
+                    job.sendWorkException(
+                        traceback.format_exc().encode('utf8'))
+            except gear.InterruptedError:
+                pass
+            except Exception:
+                self.log.exception("Exception while getting job")
 
-        delivery = request.headers.get('X-GitHub-Delivery')
+    def handle_payload(self, args):
+        headers = args.get("headers")
+        body = args.get("body")
+
+        delivery = headers.get('X-GitHub-Delivery')
         self.log.debug("Github Webhook Received: {delivery}".format(
             delivery=delivery))
 
-        self._validate_signature(request)
         # TODO(jlk): Validate project in the request is a project we know
 
         try:
-            self.__dispatch_event(request)
+            self.__dispatch_event(body, headers)
+            output = {'return_code': 200}
         except Exception:
+            output = {'return_code': 503}
             self.log.exception("Exception handling Github event:")
 
-    def __dispatch_event(self, request):
+        return output
+
+    def __dispatch_event(self, body, headers):
         try:
-            event = request.headers['X-Github-Event']
+            event = headers['x-github-event']
             self.log.debug("X-Github-Event: " + event)
         except KeyError:
             self.log.debug("Request headers missing the X-Github-Event.")
-            raise webob.exc.HTTPBadRequest('Please specify a X-Github-Event '
-                                           'header.')
+            raise Exception('Please specify a X-Github-Event header.')
 
         try:
-            json_body = request.json_body
-            self.connection.addEvent(json_body, event)
+            self.connection.addEvent(body, event)
         except Exception:
             message = 'Exception deserializing JSON body'
             self.log.exception(message)
-            raise webob.exc.HTTPBadRequest(message)
+            # TODO(jlk): Raise this as something different?
+            raise Exception(message)
 
-    def _validate_signature(self, request):
-        secret = self.connection.connection_config.get('webhook_token', None)
-        if secret is None:
-            raise RuntimeError("webhook_token is required")
+    def start(self):
+        self._running = True
+        server = self.config.get('gearman', 'server')
+        port = get_default(self.config, 'gearman', 'port', 4730)
+        ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+        ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+        ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
+        self.gearman = gear.TextWorker('Zuul Github Connector')
+        self.log.debug("Connect to gearman")
+        self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
+        self.log.debug("Waiting for server")
+        self.gearman.waitForServer()
+        self.log.debug("Registering")
+        for job in self.jobs:
+            self.gearman.registerFunction(job)
+        self.thread.start()
 
-        body = request.body
-        try:
-            request_signature = request.headers['X-Hub-Signature']
-        except KeyError:
-            raise webob.exc.HTTPUnauthorized(
-                'Please specify a X-Hub-Signature header with secret.')
-
-        payload_signature = _sign_request(body, secret)
-
-        self.log.debug("Payload Signature: {0}".format(str(payload_signature)))
-        self.log.debug("Request Signature: {0}".format(str(request_signature)))
-        if not hmac.compare_digest(
-            str(payload_signature), str(request_signature)):
-            raise webob.exc.HTTPUnauthorized(
-                'Request signature does not match calculated payload '
-                'signature. Check that secret is correct.')
-
-        return True
+    def stop(self):
+        self._running = False
+        self.gearman.stopWaitingForJobs()
+        # We join here to avoid whitelisting the thread -- if it takes more
+        # than 5s to stop in tests, there's a problem.
+        self.thread.join(timeout=5)
+        self.gearman.shutdown()
 
 
 class GithubEventConnector(threading.Thread):
@@ -343,7 +378,7 @@
         if login:
             # TODO(tobiash): it might be better to plumb in the installation id
             project = body.get('repository', {}).get('full_name')
-            return self.connection.getUser(login, project=project)
+            return self.connection.getUser(login, project)
 
     def run(self):
         while True:
@@ -360,10 +395,11 @@
 class GithubUser(collections.Mapping):
     log = logging.getLogger('zuul.GithubUser')
 
-    def __init__(self, github, username):
-        self._github = github
+    def __init__(self, username, connection, project):
+        self._connection = connection
         self._username = username
         self._data = None
+        self._project = project
 
     def __getitem__(self, key):
         self._init_data()
@@ -379,9 +415,10 @@
 
     def _init_data(self):
         if self._data is None:
-            user = self._github.user(self._username)
+            github = self._connection.getGithubClient(self._project)
+            user = github.user(self._username)
             self.log.debug("Initialized data for user %s", self._username)
-            log_rate_limit(self.log, self._github)
+            log_rate_limit(self.log, github)
             self._data = {
                 'username': user.login,
                 'name': user.name,
@@ -421,6 +458,7 @@
         self._github = None
         self.app_id = None
         self.app_key = None
+        self.sched = None
 
         self.installation_map = {}
         self.installation_token_cache = {}
@@ -456,15 +494,18 @@
             re.MULTILINE | re.IGNORECASE)
 
     def onLoad(self):
-        webhook_listener = GithubWebhookListener(self)
-        self.registerHttpHandler(self.payload_path,
-                                 webhook_listener.handle_request)
+        self.log.info('Starting GitHub connection: %s' % self.connection_name)
+        self.gearman_worker = GithubGearmanWorker(self)
+        self.log.info('Authing to GitHub')
         self._authenticateGithubAPI()
         self._prime_installation_map()
+        self.log.info('Starting event connector')
         self._start_event_connector()
+        self.log.info('Starting GearmanWorker')
+        self.gearman_worker.start()
 
     def onStop(self):
-        self.unregisterHttpHandler(self.payload_path)
+        self.gearman_worker.stop()
         self._stop_event_connector()
 
     def _start_event_connector(self):
@@ -681,7 +722,8 @@
             change.newrev = event.newrev
             change.url = self.getGitwebUrl(project, sha=event.newrev)
             change.source_event = event
-            change.files = self.getPushedFileNames(event)
+            if hasattr(event, 'commits'):
+                change.files = self.getPushedFileNames(event)
         return change
 
     def _getChange(self, project, number, patchset=None, refresh=False):
@@ -722,10 +764,10 @@
             # installation -- change queues aren't likely to span more
             # than one installation.
             for project in projects:
-                installation_id = self.installation_map.get(project)
+                installation_id = self.installation_map.get(project.name)
                 if installation_id not in installation_ids:
                     installation_ids.add(installation_id)
-                    installation_projects.add(project)
+                    installation_projects.add(project.name)
         else:
             # We aren't in the context of a change queue and we just
             # need to query all installations.  This currently only
@@ -787,7 +829,8 @@
         change.updated_at = self._ghTimestampToDate(
             change.pr.get('updated_at'))
 
-        self.sched.onChangeUpdated(change)
+        if self.sched:
+            self.sched.onChangeUpdated(change)
 
         return change
 
@@ -972,8 +1015,8 @@
         log_rate_limit(self.log, github)
         return reviews
 
-    def getUser(self, login, project=None):
-        return GithubUser(self.getGithubClient(project), login)
+    def getUser(self, login, project):
+        return GithubUser(login, self, project)
 
     def getUserUri(self, login):
         return 'https://%s/%s' % (self.server, login)
@@ -1098,6 +1141,69 @@
 
         return statuses
 
+    def getWebHandlers(self, zuul_web):
+        return [GithubWebhookHandler(self, zuul_web, 'POST', 'payload')]
+
+    def validateWebConfig(self, config, connections):
+        if 'webhook_token' not in self.connection_config:
+            raise Exception(
+                "webhook_token not found in config for connection %s" %
+                self.connection_name)
+        return True
+
+
+class GithubWebhookHandler(BaseDriverWebHandler):
+
+    log = logging.getLogger("zuul.GithubWebhookHandler")
+
+    def __init__(self, connection, zuul_web, method, path):
+        super(GithubWebhookHandler, self).__init__(
+            connection=connection, zuul_web=zuul_web, method=method, path=path)
+        self.token = self.connection.connection_config.get('webhook_token')
+
+    def _validate_signature(self, body, headers):
+        try:
+            request_signature = headers['x-hub-signature']
+        except KeyError:
+            raise web.HTTPUnauthorized(
+                reason='X-Hub-Signature header missing.')
+
+        payload_signature = _sign_request(body, self.token)
+
+        self.log.debug("Payload Signature: {0}".format(str(payload_signature)))
+        self.log.debug("Request Signature: {0}".format(str(request_signature)))
+        if not hmac.compare_digest(
+            str(payload_signature), str(request_signature)):
+            raise web.HTTPUnauthorized(
+                reason=('Request signature does not match calculated payload '
+                        'signature. Check that secret is correct.'))
+
+        return True
+
+    async def handleRequest(self, request):
+        # Note(tobiash): We need to normalize the headers. Otherwise we will
+        # have trouble to get them from the dict afterwards.
+        # e.g.
+        # GitHub: sent: X-GitHub-Event received: X-GitHub-Event
+        # urllib: sent: X-GitHub-Event received: X-Github-Event
+        #
+        # We cannot easily solve this mismatch as every http processing lib
+        # modifies the header casing in its own way and by specification http
+        # headers are case insensitive so just lowercase all so we don't have
+        # to take care later.
+        headers = dict()
+        for key, value in request.headers.items():
+            headers[key.lower()] = value
+        body = await request.read()
+        self._validate_signature(body, headers)
+        # We cannot send the raw body through gearman, so it's easy to just
+        # encode it as json, after decoding it as utf-8
+        json_body = json.loads(body.decode('utf-8'))
+        job = self.zuul_web.rpc.submitJob(
+            'github:%s:payload' % self.connection.connection_name,
+            {'headers': headers, 'body': json_body})
+        return web.json_response(json.loads(job.data[0]))
+
 
 def _status_as_tuple(status):
     """Translate a status into a tuple of user, context, state"""
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 848ae1b..57b594b 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -105,8 +105,8 @@
         url_pattern = self.config.get('status-url')
         if not url_pattern:
             sched_config = self.connection.sched.config
-            if sched_config.has_option('webapp', 'status_url'):
-                url_pattern = sched_config.get('webapp', 'status_url')
+            if sched_config.has_option('web', 'status_url'):
+                url_pattern = sched_config.get('web', 'status_url')
         url = item.formatUrlPattern(url_pattern) if url_pattern else ''
 
         description = '%s status: %s' % (item.pipeline.name,
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
index 33f8f7c..6f9b14d 100644
--- a/zuul/driver/github/githubsource.py
+++ b/zuul/driver/github/githubsource.py
@@ -144,6 +144,9 @@
         )
         return [f]
 
+    def getRefForChange(self, change):
+        return "refs/pull/%s/head" % change
+
 
 review = v.Schema({'username': str,
                    'email': str,
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 715d72b..501a2c5 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -14,14 +14,19 @@
 
 import logging
 
+from aiohttp import web
 import alembic
 import alembic.command
 import alembic.config
 import sqlalchemy as sa
 import sqlalchemy.pool
-import voluptuous as v
+from sqlalchemy.sql import select
+import urllib.parse
+import voluptuous
 
 from zuul.connection import BaseConnection
+from zuul.lib.config import get_default
+from zuul.web.handler import BaseWebHandler, StaticHandler
 
 BUILDSET_TABLE = 'zuul_buildset'
 BUILD_TABLE = 'zuul_build'
@@ -120,7 +125,122 @@
 
         return zuul_buildset_table, zuul_build_table
 
+    def getWebHandlers(self, zuul_web):
+        return [
+            SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds.json'),
+            StaticHandler(zuul_web, '/{tenant}/builds.html'),
+        ]
+
+    def validateWebConfig(self, config, connections):
+        sql_conn_name = get_default(config, 'web', 'sql_connection_name')
+        if sql_conn_name:
+            # The config wants a specific sql connection. Check the whole
+            # list of connections to make sure it can be satisfied.
+            sql_conn = connections.connections.get(sql_conn_name)
+            if not sql_conn:
+                raise Exception(
+                    "Couldn't find sql connection '%s'" % sql_conn_name)
+            if self.connection_name == sql_conn.connection_name:
+                return True
+        else:
+            # Check to see if there is more than one connection
+            conn_objects = [c for c in connections.connections.values()
+                            if isinstance(c, SQLConnection)]
+            if len(conn_objects) > 1:
+                raise Exception("Multiple sql connection found, "
+                                "set the sql_connection_name option "
+                                "in zuul.conf [web] section")
+            return True
+
+
+class SqlWebHandler(BaseWebHandler):
+    log = logging.getLogger("zuul.web.SqlHandler")
+    filters = ("project", "pipeline", "change", "patchset", "ref",
+               "result", "uuid", "job_name", "voting", "node_name", "newrev")
+
+    def __init__(self, connection, zuul_web, method, path):
+        super(SqlWebHandler, self).__init__(
+            connection=connection, zuul_web=zuul_web, method=method, path=path)
+
+    def query(self, args):
+        build = self.connection.zuul_build_table
+        buildset = self.connection.zuul_buildset_table
+        query = select([
+            buildset.c.project,
+            buildset.c.pipeline,
+            buildset.c.change,
+            buildset.c.patchset,
+            buildset.c.ref,
+            buildset.c.newrev,
+            buildset.c.ref_url,
+            build.c.result,
+            build.c.uuid,
+            build.c.job_name,
+            build.c.voting,
+            build.c.node_name,
+            build.c.start_time,
+            build.c.end_time,
+            build.c.log_url]).select_from(build.join(buildset))
+        for table in ('build', 'buildset'):
+            for key, val in args['%s_filters' % table].items():
+                if table == 'build':
+                    column = build.c
+                else:
+                    column = buildset.c
+                query = query.where(getattr(column, key).in_(val))
+        return query.limit(args['limit']).offset(args['skip']).order_by(
+            build.c.id.desc())
+
+    async def get_builds(self, args):
+        """Return a list of build"""
+        builds = []
+        with self.connection.engine.begin() as conn:
+            query = self.query(args)
+            for row in conn.execute(query):
+                build = dict(row)
+                # Convert date to iso format
+                if row.start_time:
+                    build['start_time'] = row.start_time.strftime(
+                        '%Y-%m-%dT%H:%M:%S')
+                if row.end_time:
+                    build['end_time'] = row.end_time.strftime(
+                        '%Y-%m-%dT%H:%M:%S')
+                # Compute run duration
+                if row.start_time and row.end_time:
+                    build['duration'] = (row.end_time -
+                                         row.start_time).total_seconds()
+                builds.append(build)
+        return builds
+
+    async def handleRequest(self, request):
+        try:
+            args = {
+                'buildset_filters': {},
+                'build_filters': {},
+                'limit': 50,
+                'skip': 0,
+            }
+            for k, v in urllib.parse.parse_qsl(request.rel_url.query_string):
+                if k in ("tenant", "project", "pipeline", "change",
+                         "patchset", "ref", "newrev"):
+                    args['buildset_filters'].setdefault(k, []).append(v)
+                elif k in ("uuid", "job_name", "voting", "node_name",
+                           "result"):
+                    args['build_filters'].setdefault(k, []).append(v)
+                elif k in ("limit", "skip"):
+                    args[k] = int(v)
+                else:
+                    raise ValueError("Unknown parameter %s" % k)
+            data = await self.get_builds(args)
+            resp = web.json_response(data)
+            resp.headers['Access-Control-Allow-Origin'] = '*'
+        except Exception as e:
+            self.log.exception("Jobs exception:")
+            resp = web.json_response({'error_description': 'Internal error'},
+                                     status=500)
+        return resp
+
 
 def getSchema():
-    sql_connection = v.Any(str, v.Schema(dict))
+    sql_connection = voluptuous.Any(str, voluptuous.Schema(dict))
     return sql_connection
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index b21a290..d561232 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -262,12 +262,6 @@
                 src_dir=os.path.join('src', p.canonical_name),
                 required=(p in required_projects),
             ))
-        # We are transitioning "projects" from a list to a dict
-        # indexed by canonical name, as it is much easier to access
-        # values in ansible.  Existing callers have been converted to
-        # "_projects" and "projects" is swapped; we will convert users
-        # back to "projects" and remove this soon.
-        zuul_params['_projects'] = zuul_params['projects']
 
         build = Build(job, uuid)
         build.parameters = params
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index eed3bad..53ef173 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -18,6 +18,7 @@
 import logging
 import multiprocessing
 import os
+import psutil
 import shutil
 import signal
 import shlex
@@ -573,6 +574,7 @@
         self.proc = None
         self.proc_lock = threading.Lock()
         self.running = False
+        self.started = False  # Whether playbooks have started running
         self.aborted = False
         self.aborted_reason = None
         self.thread = None
@@ -713,7 +715,7 @@
                                            project['default_branch'])
             # Update the inventory variables to indicate the ref we
             # checked out
-            p = args['zuul']['_projects'][project['canonical_name']]
+            p = args['zuul']['projects'][project['canonical_name']]
             p['checkout'] = selected
         # Delete the origin remote from each repo we set up since
         # it will not be valid within the jobs.
@@ -787,8 +789,17 @@
             return False
         if not ret:  # merge conflict
             result = dict(result='MERGER_FAILURE')
+            if self.executor_server.statsd:
+                base_key = ("zuul.executor.%s.merger" %
+                            self.executor_server.hostname)
+                self.executor_server.statsd.incr(base_key + ".FAILURE")
             self.job.sendWorkComplete(json.dumps(result))
             return False
+
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.merger" %
+                        self.executor_server.hostname)
+            self.executor_server.statsd.incr(base_key + ".SUCCESS")
         recent = ret[3]
         for key, commit in recent.items():
             (connection, project, branch) = key
@@ -842,6 +853,13 @@
         repo.checkout(selected_ref)
         return selected_ref
 
+    def getAnsibleTimeout(self, start, timeout):
+        if timeout is not None:
+            now = time.time()
+            elapsed = now - start
+            timeout = timeout - elapsed
+        return timeout
+
     def runPlaybooks(self, args):
         result = None
 
@@ -858,10 +876,16 @@
 
         pre_failed = False
         success = False
+        self.started = True
+        time_started = time.time()
+        # timeout value is total job timeout or put another way
+        # the cummulative time that pre, run, and post can consume.
+        job_timeout = args['timeout']
         for index, playbook in enumerate(self.jobdir.pre_playbooks):
             # TODOv3(pabelanger): Implement pre-run timeout setting.
+            ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout)
             pre_status, pre_code = self.runAnsiblePlaybook(
-                playbook, args['timeout'], phase='pre', index=index)
+                playbook, ansible_timeout, phase='pre', index=index)
             if pre_status != self.RESULT_NORMAL or pre_code != 0:
                 # These should really never fail, so return None and have
                 # zuul try again
@@ -869,8 +893,9 @@
                 break
 
         if not pre_failed:
+            ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout)
             job_status, job_code = self.runAnsiblePlaybook(
-                self.jobdir.playbook, args['timeout'], phase='run')
+                self.jobdir.playbook, ansible_timeout, phase='run')
             if job_status == self.RESULT_ABORTED:
                 return 'ABORTED'
             elif job_status == self.RESULT_TIMED_OUT:
@@ -891,8 +916,9 @@
 
         for index, playbook in enumerate(self.jobdir.post_playbooks):
             # TODOv3(pabelanger): Implement post-run timeout setting.
+            ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout)
             post_status, post_code = self.runAnsiblePlaybook(
-                playbook, args['timeout'], success, phase='post', index=index)
+                playbook, ansible_timeout, success, phase='post', index=index)
             if post_status == self.RESULT_ABORTED:
                 return 'ABORTED'
             if post_status != self.RESULT_NORMAL or post_code != 0:
@@ -1270,6 +1296,9 @@
             config.write('internal_poll_interval = 0.01\n')
 
             config.write('[ssh_connection]\n')
+            # NOTE(pabelanger): Try up to 3 times to run a task on a host, this
+            # helps to mitigate UNREACHABLE host errors with SSH.
+            config.write('retries = 3\n')
             # NB: when setting pipelining = True, keep_remote_files
             # must be False (the default).  Otherwise it apparently
             # will override the pipelining option and effectively
@@ -1468,6 +1497,11 @@
             wrapped=False)
         self.log.debug("Ansible complete, result %s code %s" % (
             self.RESULT_MAP[result], code))
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.phase.setup" %
+                        self.executor_server.hostname)
+            self.executor_server.statsd.incr(base_key + ".%s" %
+                                             self.RESULT_MAP[result])
         return result, code
 
     def runAnsibleCleanup(self, playbook):
@@ -1488,6 +1522,11 @@
             wrapped=False)
         self.log.debug("Ansible complete, result %s code %s" % (
             self.RESULT_MAP[result], code))
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.phase.cleanup" %
+                        self.executor_server.hostname)
+            self.executor_server.statsd.incr(base_key + ".%s" %
+                                             self.RESULT_MAP[result])
         return result, code
 
     def emitPlaybookBanner(self, playbook, step, phase, result=None):
@@ -1557,6 +1596,11 @@
             cmd=cmd, timeout=timeout, playbook=playbook)
         self.log.debug("Ansible complete, result %s code %s" % (
             self.RESULT_MAP[result], code))
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.phase.%s" %
+                        (self.executor_server.hostname, phase or 'unknown'))
+            self.executor_server.statsd.incr(base_key + ".%s" %
+                                             self.RESULT_MAP[result])
 
         self.emitPlaybookBanner(playbook, 'END', phase, result=result)
         return result, code
@@ -1603,9 +1647,10 @@
         # TODOv3(mordred): make the executor name more unique --
         # perhaps hostname+pid.
         self.hostname = get_default(self.config, 'executor', 'hostname',
-                                    socket.gethostname())
+                                    socket.getfqdn())
         self.log_streaming_port = log_streaming_port
         self.merger_lock = threading.Lock()
+        self.governor_lock = threading.Lock()
         self.run_lock = threading.Lock()
         self.verbose = False
         self.command_map = dict(
@@ -1637,6 +1682,10 @@
         load_multiplier = float(get_default(self.config, 'executor',
                                             'load_multiplier', '2.5'))
         self.max_load_avg = multiprocessing.cpu_count() * load_multiplier
+        self.max_starting_builds = self.max_load_avg * 2
+        self.min_starting_builds = max(int(multiprocessing.cpu_count() / 2), 1)
+        self.min_avail_mem = float(get_default(self.config, 'executor',
+                                               'min_avail_mem', '5.0'))
         self.accepting_work = False
         self.execution_wrapper = connections.drivers[execution_wrapper_name]
 
@@ -1754,6 +1803,10 @@
         if self._running:
             self.accepting_work = True
             self.executor_worker.registerFunction("executor:execute")
+            # TODO(jeblair): Update geard to send a noop after
+            # registering for a job which is in the queue, then remove
+            # this API violation.
+            self.executor_worker._sendGrabJobUniq()
 
     def unregister_work(self):
         self.accepting_work = False
@@ -1803,6 +1856,7 @@
         if self.statsd:
             base_key = 'zuul.executor.%s' % self.hostname
             self.statsd.gauge(base_key + '.load_average', 0)
+            self.statsd.gauge(base_key + '.pct_available_ram', 0)
             self.statsd.gauge(base_key + '.running_builds', 0)
 
         self.log.debug("Stopped")
@@ -1892,22 +1946,21 @@
                 self.log.exception("Exception while getting job")
 
     def mergerJobDispatch(self, job):
-        with self.run_lock:
-            if job.name == 'merger:cat':
-                self.log.debug("Got cat job: %s" % job.unique)
-                self.cat(job)
-            elif job.name == 'merger:merge':
-                self.log.debug("Got merge job: %s" % job.unique)
-                self.merge(job)
-            elif job.name == 'merger:refstate':
-                self.log.debug("Got refstate job: %s" % job.unique)
-                self.refstate(job)
-            elif job.name == 'merger:fileschanges':
-                self.log.debug("Got fileschanges job: %s" % job.unique)
-                self.fileschanges(job)
-            else:
-                self.log.error("Unable to handle job %s" % job.name)
-                job.sendWorkFail()
+        if job.name == 'merger:cat':
+            self.log.debug("Got cat job: %s" % job.unique)
+            self.cat(job)
+        elif job.name == 'merger:merge':
+            self.log.debug("Got merge job: %s" % job.unique)
+            self.merge(job)
+        elif job.name == 'merger:refstate':
+            self.log.debug("Got refstate job: %s" % job.unique)
+            self.refstate(job)
+        elif job.name == 'merger:fileschanges':
+            self.log.debug("Got fileschanges job: %s" % job.unique)
+            self.fileschanges(job)
+        else:
+            self.log.error("Unable to handle job %s" % job.name)
+            job.sendWorkFail()
 
     def run_executor(self):
         self.log.debug("Starting executor listener")
@@ -1946,9 +1999,10 @@
             self.statsd.incr(base_key + '.builds')
         self.job_workers[job.unique] = self._job_class(self, job)
         self.job_workers[job.unique].run()
+        self.manageLoad()
 
     def run_governor(self):
-        while not self.governor_stop_event.wait(30):
+        while not self.governor_stop_event.wait(10):
             try:
                 self.manageLoad()
             except Exception:
@@ -1956,26 +2010,57 @@
 
     def manageLoad(self):
         ''' Apply some heuristics to decide whether or not we should
-            be askign for more jobs '''
+            be asking for more jobs '''
+        with self.governor_lock:
+            return self._manageLoad()
+
+    def _manageLoad(self):
         load_avg = os.getloadavg()[0]
+        avail_mem_pct = 100.0 - psutil.virtual_memory().percent
+        starting_builds = 0
+        for worker in self.job_workers.values():
+            if not worker.started:
+                starting_builds += 1
+        max_starting_builds = max(
+            self.max_starting_builds - len(self.job_workers),
+            self.min_starting_builds)
         if self.accepting_work:
             # Don't unregister if we don't have any active jobs.
-            if load_avg > self.max_load_avg and self.job_workers:
+            if load_avg > self.max_load_avg:
                 self.log.info(
                     "Unregistering due to high system load {} > {}".format(
                         load_avg, self.max_load_avg))
                 self.unregister_work()
-        elif load_avg <= self.max_load_avg:
+            elif avail_mem_pct < self.min_avail_mem:
+                self.log.info(
+                    "Unregistering due to low memory {:3.1f}% < {}".format(
+                        avail_mem_pct, self.min_avail_mem))
+                self.unregister_work()
+            elif starting_builds >= max_starting_builds:
+                self.log.info(
+                    "Unregistering due to too many starting builds {} >= {}"
+                    .format(starting_builds, max_starting_builds))
+                self.unregister_work()
+        elif (load_avg <= self.max_load_avg and
+              avail_mem_pct >= self.min_avail_mem and
+              starting_builds < max_starting_builds):
             self.log.info(
-                "Re-registering as load is within limits {} <= {}".format(
-                    load_avg, self.max_load_avg))
+                "Re-registering as job is within limits "
+                "{} <= {} {:3.1f}% <= {} {} < {}".format(
+                    load_avg, self.max_load_avg,
+                    avail_mem_pct, self.min_avail_mem,
+                    starting_builds, max_starting_builds))
             self.register_work()
         if self.statsd:
             base_key = 'zuul.executor.%s' % self.hostname
             self.statsd.gauge(base_key + '.load_average',
                               int(load_avg * 100))
+            self.statsd.gauge(base_key + '.pct_available_ram',
+                              int(avail_mem_pct * 100))
             self.statsd.gauge(base_key + '.running_builds',
                               len(self.job_workers))
+            self.statsd.gauge(base_key + '.starting_builds',
+                              starting_builds)
 
     def finishJob(self, unique):
         del(self.job_workers[unique])
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 3b3f1ae..995eeb7 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -66,13 +66,6 @@
             if load:
                 connection.onLoad()
 
-    def registerWebapp(self, webapp):
-        for driver_name, driver in self.drivers.items():
-            if hasattr(driver, 'registerWebapp'):
-                driver.registerWebapp(webapp)
-        for connection_name, connection in self.connections.items():
-            connection.registerWebapp(webapp)
-
     def reconfigureDrivers(self, tenant):
         for driver in self.drivers.values():
             if hasattr(driver, 'reconfigure'):
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 3c77990..88ddf7d 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -722,9 +722,10 @@
         return True
 
     def onBuildCompleted(self, build):
-        self.log.debug("Build %s completed" % build)
         item = build.build_set.item
 
+        self.log.debug("Build %s of %s completed" % (build, item.change))
+
         item.setResult(build)
         item.pipeline.layout.tenant.semaphore_handler.release(item, build.job)
         self.log.debug("Item %s status is now:\n %s" %
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index bd4ca58..5e102b4 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -17,6 +17,7 @@
 import logging
 import os
 import shutil
+import time
 
 import git
 import gitdb
@@ -52,14 +53,10 @@
         raise
 
 
-class ZuulReference(git.Reference):
-    _common_path_default = "refs/zuul"
-    _points_to_commits_only = True
-
-
 class Repo(object):
     def __init__(self, remote, local, email, username, speed_limit, speed_time,
-                 sshkey=None, cache_path=None, logger=None, git_timeout=300):
+                 sshkey=None, cache_path=None, logger=None, git_timeout=300,
+                 retry_attempts=3, retry_interval=30):
         if logger is None:
             self.log = logging.getLogger("zuul.Repo")
         else:
@@ -78,6 +75,8 @@
         self.username = username
         self.cache_path = cache_path
         self._initialized = False
+        self.retry_attempts = retry_attempts
+        self.retry_interval = retry_interval
         try:
             self._ensure_cloned()
         except Exception:
@@ -123,14 +122,37 @@
     def _git_clone(self, url):
         mygit = git.cmd.Git(os.getcwd())
         mygit.update_environment(**self.env)
-        with timeout_handler(self.local_path):
-            mygit.clone(git.cmd.Git.polish_url(url), self.local_path,
-                        kill_after_timeout=self.git_timeout)
+
+        for attempt in range(1, self.retry_attempts + 1):
+            try:
+                with timeout_handler(self.local_path):
+                    mygit.clone(git.cmd.Git.polish_url(url), self.local_path,
+                                kill_after_timeout=self.git_timeout)
+                break
+            except Exception as e:
+                if attempt < self.retry_attempts:
+                    time.sleep(self.retry_interval)
+                    self.log.warning("Retry %s: Clone %s" % (
+                        attempt, self.local_path))
+                else:
+                    raise
 
     def _git_fetch(self, repo, remote, ref=None, **kwargs):
-        with timeout_handler(self.local_path):
-            repo.git.fetch(remote, ref, kill_after_timeout=self.git_timeout,
-                           **kwargs)
+        for attempt in range(1, self.retry_attempts + 1):
+            try:
+                with timeout_handler(self.local_path):
+                    repo.git.fetch(remote, ref,
+                                   kill_after_timeout=self.git_timeout,
+                                   **kwargs)
+                break
+            except Exception as e:
+                if attempt < self.retry_attempts:
+                    time.sleep(self.retry_interval)
+                    self.log.exception("Retry %s: Fetch %s %s %s" % (
+                        attempt, self.local_path, remote, ref))
+                    self._ensure_cloned()
+                else:
+                    raise
 
     def createRepoObject(self):
         self._ensure_cloned()
@@ -143,19 +165,30 @@
         self.update()
         repo = self.createRepoObject()
         origin = repo.remotes.origin
+        seen = set()
+        head = None
+        stale_refs = origin.stale_refs
+        # Update our local heads to match the remote, and pick one to
+        # reset the repo to.  We don't delete anything at this point
+        # because we want to make sure the repo is in a state stable
+        # enough for git to operate.
         for ref in origin.refs:
             if ref.remote_head == 'HEAD':
                 continue
+            if ref in stale_refs:
+                continue
             repo.create_head(ref.remote_head, ref, force=True)
-
-        # try reset to remote HEAD (usually origin/master)
-        # If it fails, pick the first reference
-        try:
-            repo.head.reference = origin.refs['HEAD']
-        except IndexError:
-            repo.head.reference = origin.refs[0]
+            seen.add(ref.remote_head)
+            if head is None:
+                head = ref.remote_head
+        self.log.debug("Reset to %s", head)
+        repo.head.reference = head
         reset_repo_to_head(repo)
         repo.git.clean('-x', '-f', '-d')
+        for ref in stale_refs:
+            self.log.debug("Delete stale ref %s", ref.remote_head)
+            repo.delete_head(ref.remote_head, force=True)
+            git.refs.RemoteReference.delete(repo, ref, force=True)
 
     def prune(self):
         repo = self.createRepoObject()
@@ -163,7 +196,7 @@
         stale_refs = origin.stale_refs
         if stale_refs:
             self.log.debug("Pruning stale refs: %s", stale_refs)
-            git.refs.RemoteReference.delete(repo, *stale_refs)
+            git.refs.RemoteReference.delete(repo, force=True, *stale_refs)
 
     def getBranchHead(self, branch):
         repo = self.createRepoObject()
@@ -193,11 +226,12 @@
         return repo.refs
 
     def setRef(self, path, hexsha, repo=None):
+        self.log.debug("Create reference %s at %s in %s",
+                       path, hexsha, self.local_path)
         if repo is None:
             repo = self.createRepoObject()
         binsha = gitdb.util.to_bin_sha(hexsha)
         obj = git.objects.Object.new_from_sha(repo, binsha)
-        self.log.debug("Create reference %s", path)
         git.refs.Reference.create(repo, path, obj, force=True)
 
     def setRefs(self, refs):
@@ -227,14 +261,6 @@
         repo.git.checkout(ref)
         return repo.head.commit
 
-    def checkoutLocalBranch(self, branch):
-        # TODO(jeblair): retire in favor of checkout
-        repo = self.createRepoObject()
-        # Perform a hard reset before checking out so that we clean up
-        # anything that might be left over from a merge.
-        reset_repo_to_head(repo)
-        repo.heads[branch].checkout()
-
     def cherryPick(self, ref):
         repo = self.createRepoObject()
         self.log.debug("Cherry-picking %s" % ref)
@@ -268,12 +294,6 @@
         repo = self.createRepoObject()
         self._git_fetch(repo, repository, ref)
 
-    def createZuulRef(self, ref, commit='HEAD'):
-        repo = self.createRepoObject()
-        self.log.debug("CreateZuulRef %s at %s on %s" % (ref, commit, repo))
-        ref = ZuulReference.create(repo, ref, commit)
-        return ref.commit
-
     def push(self, local, remote):
         repo = self.createRepoObject()
         self.log.debug("Pushing %s:%s to %s" % (local, remote,
@@ -504,20 +524,6 @@
             return None
         # Store this commit as the most recent for this project-branch
         recent[key] = commit
-        # Set the Zuul ref for this item to point to the most recent
-        # commits of each project-branch
-        for key, mrc in recent.items():
-            connection, project, branch = key
-            zuul_ref = None
-            try:
-                repo = self.getRepo(connection, project)
-                zuul_ref = branch + '/' + item['buildset_uuid']
-                if not repo.getCommitFromRef(zuul_ref):
-                    repo.createZuulRef(zuul_ref, mrc)
-            except Exception:
-                self.log.exception("Unable to set zuul ref %s for "
-                                   "item %s" % (zuul_ref, item))
-                return None
         return commit
 
     def mergeChanges(self, items, files=None, dirs=None, repo_state=None):
diff --git a/zuul/model.py b/zuul/model.py
index 29c5a9d..38f2d6b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -476,8 +476,9 @@
     or they may appears anonymously in in-line job definitions.
     """
 
-    def __init__(self, name=None):
+    def __init__(self, name=None, source_context=None):
         self.name = name or ''
+        self.source_context = source_context
         self.nodes = OrderedDict()
         self.groups = OrderedDict()
 
@@ -612,6 +613,9 @@
                 self.source_context == other.source_context and
                 self.secret_data == other.secret_data)
 
+    def areDataEqual(self, other):
+        return (self.secret_data == other.secret_data)
+
     def __repr__(self):
         return '<Secret %s>' % (self.name,)
 
@@ -662,7 +666,6 @@
         if not isinstance(other, SourceContext):
             return False
         return (self.project == other.project and
-                self.branch == other.branch and
                 self.trusted == other.trusted)
 
     def __ne__(self, other):
@@ -1057,7 +1060,8 @@
                                         "from other projects."
                                         % (repr(self), this_origin))
                 if k not in set(['pre_run', 'run', 'post_run', 'roles',
-                                 'variables', 'required_projects']):
+                                 'variables', 'required_projects',
+                                 'allowed_projects']):
                     # TODO(jeblair): determine if deepcopy is required
                     setattr(self, k, copy.deepcopy(other._get(k)))
 
@@ -1094,6 +1098,12 @@
             self.updateVariables(other.variables)
         if other._get('required_projects') is not None:
             self.updateProjects(other.required_projects)
+        if (other._get('allowed_projects') is not None and
+            self._get('allowed_projects') is not None):
+            self.allowed_projects = self.allowed_projects.intersection(
+                other.allowed_projects)
+        elif other._get('allowed_projects') is not None:
+            self.allowed_projects = copy.deepcopy(other.allowed_projects)
 
         for k in self.context_attributes:
             if (other._get(k) is not None and
@@ -1105,8 +1115,18 @@
 
         self.inheritance_path = self.inheritance_path + (repr(other),)
 
-    def changeMatches(self, change):
-        if self.branch_matcher and not self.branch_matcher.matches(change):
+    def changeMatches(self, change, override_branch=None):
+        if override_branch is None:
+            branch_change = change
+        else:
+            # If an override branch is supplied, create a very basic
+            # change (a Ref) and set its branch to the override
+            # branch.
+            branch_change = Ref(change.project)
+            branch_change.ref = override_branch
+
+        if self.branch_matcher and not self.branch_matcher.matches(
+                branch_change):
             return False
 
         if self.file_matcher and not self.file_matcher.matches(change):
@@ -2068,9 +2088,6 @@
     def isUpdateOf(self, other):
         return False
 
-    def filterJobs(self, jobs):
-        return filter(lambda job: job.changeMatches(self), jobs)
-
     def getRelatedChanges(self):
         return set()
 
@@ -2532,6 +2549,7 @@
         # that override some attribute of the job.  These aspects all
         # inherit from the reference definition.
         noop = Job('noop')
+        noop.description = 'A job that will always succeed, no operation.'
         noop.parent = noop.BASE_JOB_MARKER
         noop.run = 'noop.yaml'
         self.jobs = {'noop': [noop]}
@@ -2584,18 +2602,65 @@
         return True
 
     def addNodeSet(self, nodeset):
-        if nodeset.name in self.nodesets:
-            raise Exception("NodeSet %s already defined" % (nodeset.name,))
+        # It's ok to have a duplicate nodeset definition, but only if
+        # they are in different branches of the same repo, and have
+        # the same values.
+        other = self.nodesets.get(nodeset.name)
+        if other is not None:
+            if not nodeset.source_context.isSameProject(other.source_context):
+                raise Exception("Nodeset %s already defined in project %s" %
+                                (nodeset.name, other.source_context.project))
+            if nodeset.source_context.branch == other.source_context.branch:
+                raise Exception("Nodeset %s already defined" % (nodeset.name,))
+            if nodeset != other:
+                raise Exception("Nodeset %s does not match existing definition"
+                                " in branch %s" %
+                                (nodeset.name, other.source_context.branch))
+            # Identical data in a different branch of the same project;
+            # ignore the duplicate definition
+            return
         self.nodesets[nodeset.name] = nodeset
 
     def addSecret(self, secret):
-        if secret.name in self.secrets:
-            raise Exception("Secret %s already defined" % (secret.name,))
+        # It's ok to have a duplicate secret definition, but only if
+        # they are in different branches of the same repo, and have
+        # the same values.
+        other = self.secrets.get(secret.name)
+        if other is not None:
+            if not secret.source_context.isSameProject(other.source_context):
+                raise Exception("Secret %s already defined in project %s" %
+                                (secret.name, other.source_context.project))
+            if secret.source_context.branch == other.source_context.branch:
+                raise Exception("Secret %s already defined" % (secret.name,))
+            if not secret.areDataEqual(other):
+                raise Exception("Secret %s does not match existing definition"
+                                " in branch %s" %
+                                (secret.name, other.source_context.branch))
+            # Identical data in a different branch of the same project;
+            # ignore the duplicate definition
+            return
         self.secrets[secret.name] = secret
 
     def addSemaphore(self, semaphore):
-        if semaphore.name in self.semaphores:
-            raise Exception("Semaphore %s already defined" % (semaphore.name,))
+        # It's ok to have a duplicate semaphore definition, but only if
+        # they are in different branches of the same repo, and have
+        # the same values.
+        other = self.semaphores.get(semaphore.name)
+        if other is not None:
+            if not semaphore.source_context.isSameProject(
+                    other.source_context):
+                raise Exception("Semaphore %s already defined in project %s" %
+                                (semaphore.name, other.source_context.project))
+            if semaphore.source_context.branch == other.source_context.branch:
+                raise Exception("Semaphore %s already defined" %
+                                (semaphore.name,))
+            if semaphore != other:
+                raise Exception("Semaphore %s does not match existing"
+                                " definition in branch %s" %
+                                (semaphore.name, other.source_context.branch))
+            # Identical data in a different branch of the same project;
+            # ignore the duplicate definition
+            return
         self.semaphores[semaphore.name] = semaphore
 
     def addPipeline(self, pipeline):
@@ -2618,21 +2683,33 @@
     def addProjectConfig(self, project_config):
         self.project_configs[project_config.name] = project_config
 
-    def collectJobs(self, item, jobname, change, path=None, jobs=None,
-                    stack=None):
-        if stack is None:
-            stack = []
-        if jobs is None:
-            jobs = []
-        if path is None:
-            path = []
-        path.append(jobname)
+    def _updateOverrideCheckouts(self, override_checkouts, job):
+        # Update the values in an override_checkouts dict with those
+        # in a job.  Used in collectJobVariants.
+        if job.override_checkout:
+            override_checkouts[None] = job.override_checkout
+        for req in job.required_projects.values():
+            if req.override_checkout:
+                override_checkouts[req.project_name] = req.override_checkout
+
+    def _collectJobVariants(self, item, jobname, change, path, jobs, stack,
+                            override_checkouts, indent):
         matched = False
-        indent = len(path) + 1
-        item.debug("Collecting job variants for {jobname}".format(
-            jobname=jobname), indent=indent)
+        local_override_checkouts = override_checkouts.copy()
+        override_branch = None
+        project = None
         for variant in self.getJobs(jobname):
-            if not variant.changeMatches(change):
+            if project is None and variant.source_context:
+                project = variant.source_context.project
+                if override_checkouts.get(None) is not None:
+                    override_branch = override_checkouts.get(None)
+                override_branch = override_checkouts.get(
+                    project.canonical_name, override_branch)
+                branches = self.tenant.getProjectBranches(project)
+                if override_branch not in branches:
+                    override_branch = None
+            if not variant.changeMatches(change,
+                                         override_branch=override_branch):
                 self.log.debug("Variant %s did not match %s", repr(variant),
                                change)
                 item.debug("Variant {variant} did not match".format(
@@ -2648,17 +2725,53 @@
                     parent = self.tenant.default_base_job
             else:
                 parent = None
+            self._updateOverrideCheckouts(local_override_checkouts, variant)
             if parent and parent not in path:
                 if parent in stack:
                     raise Exception("Dependency cycle in jobs: %s" % stack)
                 self.collectJobs(item, parent, change, path, jobs,
-                                 stack + [jobname])
+                                 stack + [jobname], local_override_checkouts)
             matched = True
-            jobs.append(variant)
+            if variant not in jobs:
+                jobs.append(variant)
+        return matched
+
+    def collectJobs(self, item, jobname, change, path=None, jobs=None,
+                    stack=None, override_checkouts=None):
+        # Stack is the recursion stack of job parent names.  Each time
+        # we go up a level, we add to stack, and it's popped as we
+        # descend.
+        if stack is None:
+            stack = []
+        # Jobs is the list of jobs we've accumulated.
+        if jobs is None:
+            jobs = []
+        # Path is the list of job names we've examined.  It
+        # accumulates and never reduces.  If more than one job has the
+        # same parent, this will prevent us from adding it a second
+        # time.
+        if path is None:
+            path = []
+        # Override_checkouts is a dictionary of canonical project
+        # names -> branch names.  It is not mutated, but instead new
+        # copies are made and updated as we ascend the hierarchy, so
+        # higher levels don't affect lower levels after we descend.
+        # It's used to override the branch matchers for jobs.
+        if override_checkouts is None:
+            override_checkouts = {}
+        path.append(jobname)
+        matched = False
+        indent = len(path) + 1
+        msg = "Collecting job variants for {jobname}".format(jobname=jobname)
+        self.log.debug(msg)
+        item.debug(msg, indent=indent)
+        matched = self._collectJobVariants(
+            item, jobname, change, path, jobs, stack, override_checkouts,
+            indent)
         if not matched:
             self.log.debug("No matching parents for job %s and change %s",
                            jobname, change)
-            item.debug("No matching parent for {jobname}".format(
+            item.debug("No matching parents for {jobname}".format(
                 jobname=repr(jobname)), indent=indent)
             raise NoMatchingParentError()
         return jobs
@@ -2673,8 +2786,17 @@
             self.log.debug("Collecting jobs %s for %s", jobname, change)
             item.debug("Freezing job {jobname}".format(
                 jobname=jobname), indent=1)
+            # Create the initial list of override_checkouts, which are
+            # used as we walk up the hierarchy to expand the set of
+            # jobs which match.
+            override_checkouts = {}
+            for variant in job_list.jobs[jobname]:
+                if variant.changeMatches(change):
+                    self._updateOverrideCheckouts(override_checkouts, variant)
             try:
-                variants = self.collectJobs(item, jobname, change)
+                variants = self.collectJobs(
+                    item, jobname, change,
+                    override_checkouts=override_checkouts)
             except NoMatchingParentError:
                 variants = None
             if not variants:
@@ -2714,7 +2836,7 @@
                 item.debug("No matching pipeline variants for {jobname}".
                            format(jobname=jobname), indent=2)
                 continue
-            if (frozen_job.allowed_projects and
+            if (frozen_job.allowed_projects is not None and
                 change.project.name not in frozen_job.allowed_projects):
                 raise Exception("Project %s is not allowed to run job %s" %
                                 (change.project.name, frozen_job.name))
@@ -2750,6 +2872,15 @@
         self.name = name
         self.max = int(max)
 
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __eq__(self, other):
+        if not isinstance(other, Semaphore):
+            return False
+        return (self.name == other.name and
+                self.max == other.max)
+
 
 class SemaphoreHandler(object):
     log = logging.getLogger("zuul.SemaphoreHandler")
diff --git a/zuul/nodepool.py b/zuul/nodepool.py
index b96d1ca..6e7064c 100644
--- a/zuul/nodepool.py
+++ b/zuul/nodepool.py
@@ -165,6 +165,7 @@
         self.log.debug("Updating node request %s" % (request,))
 
         if request.uid not in self.requests:
+            self.log.debug("Request %s is unknown" % (request.uid,))
             return False
 
         if request.canceled:
@@ -193,14 +194,21 @@
 
     def acceptNodes(self, request, request_id):
         # Called by the scheduler when it wants to accept and lock
-        # nodes for (potential) use.
+        # nodes for (potential) use.  Return False if there is a
+        # problem with the request (canceled or retrying), True if it
+        # is ready to be acted upon (success or failure).
 
         self.log.info("Accepting node request %s" % (request,))
 
         if request_id != request.id:
             self.log.info("Skipping node accept for %s (resubmitted as %s)",
                           request_id, request.id)
-            return
+            return False
+
+        if request.canceled:
+            self.log.info("Ignoring canceled node request %s" % (request,))
+            # The request was already deleted when it was canceled
+            return False
 
         # Make sure the request still exists. It's possible it could have
         # disappeared if we lost the ZK session between when the fulfillment
@@ -208,13 +216,13 @@
         # processing it. Nodepool will automatically reallocate the assigned
         # nodes in that situation.
         if not self.sched.zk.nodeRequestExists(request):
-            self.log.info("Request %s no longer exists", request.id)
-            return
-
-        if request.canceled:
-            self.log.info("Ignoring canceled node request %s" % (request,))
-            # The request was already deleted when it was canceled
-            return
+            self.log.info("Request %s no longer exists, resubmitting",
+                          request.id)
+            request.id = None
+            request.state = model.STATE_REQUESTED
+            self.requests[request.uid] = request
+            self.sched.zk.submitNodeRequest(request, self._updateNodeRequest)
+            return False
 
         locked = False
         if request.fulfilled:
@@ -239,3 +247,4 @@
             # them.
             if locked:
                 self.unlockNodeSet(request.nodeset)
+        return True
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 1bff5cb..e05ee06 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -75,7 +75,7 @@
 
     def _formatItemReportStart(self, item, with_jobs=True):
         status_url = get_default(self.connection.sched.config,
-                                 'webapp', 'status_url', '')
+                                 'web', 'status_url', '')
         return item.pipeline.start_message.format(pipeline=item.pipeline,
                                                   status_url=status_url)
 
diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py
index 8f2e5dc..a947ed0 100644
--- a/zuul/rpcclient.py
+++ b/zuul/rpcclient.py
@@ -48,10 +48,12 @@
         self.log.debug("Job complete, success: %s" % (not job.failure))
         return job
 
-    def autohold(self, tenant, project, job, reason, count):
+    def autohold(self, tenant, project, job, change, ref, reason, count):
         data = {'tenant': tenant,
                 'project': project,
                 'job': job,
+                'change': change,
+                'ref': ref,
                 'reason': reason,
                 'count': count}
         return not self.submitJob('zuul:autohold', data).failure
diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py
index e5016df..f3f55f6 100644
--- a/zuul/rpclistener.py
+++ b/zuul/rpclistener.py
@@ -150,7 +150,20 @@
             job.sendWorkException(error.encode('utf8'))
             return
 
+        if args['change'] and args['ref']:
+            job.sendWorkException("Change and ref can't be both used "
+                                  "for the same request")
+
+        if args['change']:
+            # Convert change into ref based on zuul connection
+            ref_filter = project.source.getRefForChange(args['change'])
+        elif args['ref']:
+            ref_filter = "%s" % args['ref']
+        else:
+            ref_filter = ".*"
+
         params['job_name'] = args['job']
+        params['ref_filter'] = ref_filter
         params['reason'] = args['reason']
 
         if args['count'] < 0:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a2e3b6e..2bce43f 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -19,6 +19,7 @@
 import logging
 import os
 import pickle
+import re
 import queue
 import socket
 import sys
@@ -214,7 +215,7 @@
     def __init__(self, config, testonly=False):
         threading.Thread.__init__(self)
         self.daemon = True
-        self.hostname = socket.gethostname()
+        self.hostname = socket.getfqdn()
         self.wake_event = threading.Event()
         self.layout_lock = threading.Lock()
         self.run_handler_lock = threading.Lock()
@@ -292,11 +293,10 @@
             except Exception:
                 self.log.exception("Exception while processing command")
 
-    def registerConnections(self, connections, webapp, load=True):
+    def registerConnections(self, connections, load=True):
         # load: whether or not to trigger the onLoad for the connection. This
         # is useful for not doing a full load during layout validation.
         self.connections = connections
-        self.connections.registerWebapp(webapp)
         self.connections.registerScheduler(self, load)
 
     def stopConnections(self):
@@ -436,8 +436,9 @@
         self.last_reconfigured = int(time.time())
         # TODOv3(jeblair): reconfigure time should be per-tenant
 
-    def autohold(self, tenant_name, project_name, job_name, reason, count):
-        key = (tenant_name, project_name, job_name)
+    def autohold(self, tenant_name, project_name, job_name, ref_filter,
+                 reason, count):
+        key = (tenant_name, project_name, job_name, ref_filter)
         if count == 0 and key in self.autohold_requests:
             self.log.debug("Removing autohold for %s", key)
             del self.autohold_requests[key]
@@ -973,6 +974,84 @@
             self.log.exception("Exception estimating build time:")
         pipeline.manager.onBuildStarted(event.build)
 
+    def _getAutoholdRequestKey(self, build):
+        change = build.build_set.item.change
+
+        autohold_key_base = (build.pipeline.layout.tenant.name,
+                             change.project.canonical_name,
+                             build.job.name)
+
+        class Scope(object):
+            """Enum defining a precedence/priority of autohold requests.
+
+            Autohold requests for specific refs should be fulfilled first,
+            before those for changes, and generic jobs.
+
+            Matching algorithm goes over all existing autohold requests, and
+            returns one with the highest number (in case of duplicated
+            requests the last one wins).
+            """
+            NONE = 0
+            JOB = 1
+            CHANGE = 2
+            REF = 3
+
+        def autohold_key_base_issubset(base, request_key):
+            """check whether the given key is a subset of the build key"""
+            index = 0
+            base_len = len(base)
+            while index < base_len:
+                if base[index] != request_key[index]:
+                    return False
+                index += 1
+            return True
+
+        # Do a partial match of the autohold key against all autohold
+        # requests, ignoring the last element of the key (ref filter),
+        # and finally do a regex match between ref filter from
+        # the autohold request and the build's change ref to check
+        # if it matches. Lastly, make sure that we match the most
+        # specific autohold request by comparing "scopes"
+        # of requests - the most specific is selected.
+        autohold_key = None
+        scope = Scope.NONE
+        for request in self.autohold_requests:
+            ref_filter = request[-1]
+            if not autohold_key_base_issubset(autohold_key_base, request) \
+                or not re.match(ref_filter, change.ref):
+                continue
+
+            if ref_filter == ".*":
+                candidate_scope = Scope.JOB
+            elif ref_filter.endswith(".*"):
+                candidate_scope = Scope.CHANGE
+            else:
+                candidate_scope = Scope.REF
+
+            if candidate_scope > scope:
+                scope = candidate_scope
+                autohold_key = request
+
+        return autohold_key
+
+    def _processAutohold(self, build):
+
+        # We explicitly only want to hold nodes for jobs if they have
+        # failed and have an autohold request.
+        if build.result != "FAILURE":
+            return
+
+        autohold_key = self._getAutoholdRequestKey(build)
+        try:
+            self.nodepool.holdNodeSet(build.nodeset, autohold_key)
+        except Exception:
+            self.log.exception("Unable to process autohold for %s:",
+                               autohold_key)
+            if autohold_key in self.autohold_requests:
+                self.log.debug("Removing autohold %s due to exception",
+                               autohold_key)
+                del self.autohold_requests[autohold_key]
+
     def _doBuildCompletedEvent(self, event):
         build = event.build
 
@@ -980,27 +1059,10 @@
         # to pass this on to the pipeline manager, make sure we return
         # the nodes to nodepool.
         try:
-            nodeset = build.nodeset
-            autohold_key = (build.pipeline.layout.tenant.name,
-                            build.build_set.item.change.project.canonical_name,
-                            build.job.name)
-            if (build.result == "FAILURE" and
-                autohold_key in self.autohold_requests):
-                # We explicitly only want to hold nodes for jobs if they have
-                # failed and have an autohold request.
-                try:
-                    self.nodepool.holdNodeSet(nodeset, autohold_key)
-                except Exception:
-                    self.log.exception("Unable to process autohold for %s:",
-                                       autohold_key)
-                    if autohold_key in self.autohold_requests:
-                        self.log.debug("Removing autohold %s due to exception",
-                                       autohold_key)
-                        del self.autohold_requests[autohold_key]
-
-            self.nodepool.returnNodeSet(nodeset)
+            self._processAutohold(build)
+            self.nodepool.returnNodeSet(build.nodeset)
         except Exception:
-            self.log.exception("Unable to return nodeset %s" % (nodeset,))
+            self.log.exception("Unable to return nodeset %s" % build.nodeset)
 
         if build.build_set is not build.build_set.item.current_build_set:
             self.log.debug("Build %s is not in the current build set" %
@@ -1036,8 +1098,8 @@
         request_id = event.request_id
         build_set = request.build_set
 
-        self.nodepool.acceptNodes(request, request_id)
-        if request.canceled:
+        ready = self.nodepool.acceptNodes(request, request_id)
+        if not ready:
             return
 
         if build_set is not build_set.item.current_build_set:
@@ -1085,6 +1147,10 @@
         pipelines = []
         data['pipelines'] = pipelines
         tenant = self.abide.tenants.get(tenant_name)
+        if not tenant:
+            self.log.warning("Tenant %s isn't loaded" % tenant_name)
+            return json.dumps(
+                {"message": "Tenant %s isn't ready" % tenant_name})
         for pipeline in tenant.layout.pipelines.values():
             pipelines.append(pipeline.formatStatusJSON(websocket_url))
         return json.dumps(data)
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index a98a6c8..adbafb5 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -20,15 +20,13 @@
 import logging
 import os
 import time
-import urllib.parse
 import uvloop
 
 import aiohttp
 from aiohttp import web
 
-from sqlalchemy.sql import select
-
 import zuul.rpcclient
+from zuul.web.handler import StaticHandler
 
 STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static')
 
@@ -161,11 +159,11 @@
             'key_get': self.key_get,
         }
 
-    def tenant_list(self, request):
+    async def tenant_list(self, request):
         job = self.rpc.submitJob('zuul:tenant_list', {})
         return web.json_response(json.loads(job.data[0]))
 
-    def status_get(self, request):
+    async def status_get(self, request):
         tenant = request.match_info["tenant"]
         if tenant not in self.cache or \
            (time.time() - self.cache_time[tenant]) > self.cache_expiry:
@@ -179,14 +177,14 @@
         resp.last_modified = self.cache_time[tenant]
         return resp
 
-    def job_list(self, request):
+    async def job_list(self, request):
         tenant = request.match_info["tenant"]
         job = self.rpc.submitJob('zuul:job_list', {'tenant': tenant})
         resp = web.json_response(json.loads(job.data[0]))
         resp.headers['Access-Control-Allow-Origin'] = '*'
         return resp
 
-    def key_get(self, request):
+    async def key_get(self, request):
         tenant = request.match_info["tenant"]
         project = request.match_info["project"]
         job = self.rpc.submitJob('zuul:key_get', {'tenant': tenant,
@@ -195,7 +193,7 @@
 
     async def processRequest(self, request, action):
         try:
-            resp = self.controllers[action](request)
+            resp = await self.controllers[action](request)
         except asyncio.CancelledError:
             self.log.debug("request handling cancelled")
         except Exception as e:
@@ -205,93 +203,6 @@
         return resp
 
 
-class SqlHandler(object):
-    log = logging.getLogger("zuul.web.SqlHandler")
-    filters = ("project", "pipeline", "change", "patchset", "ref",
-               "result", "uuid", "job_name", "voting", "node_name", "newrev")
-
-    def __init__(self, connection):
-        self.connection = connection
-
-    def query(self, args):
-        build = self.connection.zuul_build_table
-        buildset = self.connection.zuul_buildset_table
-        query = select([
-            buildset.c.project,
-            buildset.c.pipeline,
-            buildset.c.change,
-            buildset.c.patchset,
-            buildset.c.ref,
-            buildset.c.newrev,
-            buildset.c.ref_url,
-            build.c.result,
-            build.c.uuid,
-            build.c.job_name,
-            build.c.voting,
-            build.c.node_name,
-            build.c.start_time,
-            build.c.end_time,
-            build.c.log_url]).select_from(build.join(buildset))
-        for table in ('build', 'buildset'):
-            for k, v in args['%s_filters' % table].items():
-                if table == 'build':
-                    column = build.c
-                else:
-                    column = buildset.c
-                query = query.where(getattr(column, k).in_(v))
-        return query.limit(args['limit']).offset(args['skip']).order_by(
-            build.c.id.desc())
-
-    def get_builds(self, args):
-        """Return a list of build"""
-        builds = []
-        with self.connection.engine.begin() as conn:
-            query = self.query(args)
-            for row in conn.execute(query):
-                build = dict(row)
-                # Convert date to iso format
-                if row.start_time:
-                    build['start_time'] = row.start_time.strftime(
-                        '%Y-%m-%dT%H:%M:%S')
-                if row.end_time:
-                    build['end_time'] = row.end_time.strftime(
-                        '%Y-%m-%dT%H:%M:%S')
-                # Compute run duration
-                if row.start_time and row.end_time:
-                    build['duration'] = (row.end_time -
-                                         row.start_time).total_seconds()
-                builds.append(build)
-        return builds
-
-    async def processRequest(self, request):
-        try:
-            args = {
-                'buildset_filters': {},
-                'build_filters': {},
-                'limit': 50,
-                'skip': 0,
-            }
-            for k, v in urllib.parse.parse_qsl(request.rel_url.query_string):
-                if k in ("tenant", "project", "pipeline", "change",
-                         "patchset", "ref", "newrev"):
-                    args['buildset_filters'].setdefault(k, []).append(v)
-                elif k in ("uuid", "job_name", "voting", "node_name",
-                           "result"):
-                    args['build_filters'].setdefault(k, []).append(v)
-                elif k in ("limit", "skip"):
-                    args[k] = int(v)
-                else:
-                    raise ValueError("Unknown parameter %s" % k)
-            data = self.get_builds(args)
-            resp = web.json_response(data)
-            resp.headers['Access-Control-Allow-Origin'] = '*'
-        except Exception as e:
-            self.log.exception("Jobs exception:")
-            resp = web.json_response({'error_description': 'Internal error'},
-                                     status=500)
-        return resp
-
-
 class ZuulWeb(object):
 
     log = logging.getLogger("zuul.web.ZuulWeb")
@@ -300,7 +211,7 @@
                  gear_server, gear_port,
                  ssl_key=None, ssl_cert=None, ssl_ca=None,
                  static_cache_expiry=3600,
-                 sql_connection=None):
+                 connections=None):
         self.listen_address = listen_address
         self.listen_port = listen_port
         self.event_loop = None
@@ -312,10 +223,10 @@
                                             ssl_key, ssl_cert, ssl_ca)
         self.log_streaming_handler = LogStreamingHandler(self.rpc)
         self.gearman_handler = GearmanHandler(self.rpc)
-        if sql_connection:
-            self.sql_handler = SqlHandler(sql_connection)
-        else:
-            self.sql_handler = None
+        self._plugin_routes = []  # type: List[zuul.web.handler.BaseWebHandler]
+        connections = connections or []
+        for connection in connections:
+            self._plugin_routes.extend(connection.getWebHandlers(self))
 
     async def _handleWebsocket(self, request):
         return await self.log_streaming_handler.processRequest(
@@ -331,30 +242,9 @@
     async def _handleJobsRequest(self, request):
         return await self.gearman_handler.processRequest(request, 'job_list')
 
-    async def _handleSqlRequest(self, request):
-        return await self.sql_handler.processRequest(request)
-
     async def _handleKeyRequest(self, request):
         return await self.gearman_handler.processRequest(request, 'key_get')
 
-    async def _handleStaticRequest(self, request):
-        fp = None
-        if request.path.endswith("tenants.html") or request.path.endswith("/"):
-            fp = os.path.join(STATIC_DIR, "index.html")
-        elif request.path.endswith("status.html"):
-            fp = os.path.join(STATIC_DIR, "status.html")
-        elif request.path.endswith("jobs.html"):
-            fp = os.path.join(STATIC_DIR, "jobs.html")
-        elif request.path.endswith("builds.html"):
-            fp = os.path.join(STATIC_DIR, "builds.html")
-        elif request.path.endswith("stream.html"):
-            fp = os.path.join(STATIC_DIR, "stream.html")
-        headers = {}
-        if self.static_cache_expiry:
-            headers['Cache-Control'] = "public, max-age=%d" % \
-                self.static_cache_expiry
-        return web.FileResponse(fp, headers=headers)
-
     def run(self, loop=None):
         """
         Run the websocket daemon.
@@ -372,18 +262,18 @@
             ('GET', '/{tenant}/jobs.json', self._handleJobsRequest),
             ('GET', '/{tenant}/console-stream', self._handleWebsocket),
             ('GET', '/{tenant}/{project:.*}.pub', self._handleKeyRequest),
-            ('GET', '/{tenant}/status.html', self._handleStaticRequest),
-            ('GET', '/{tenant}/jobs.html', self._handleStaticRequest),
-            ('GET', '/{tenant}/stream.html', self._handleStaticRequest),
-            ('GET', '/tenants.html', self._handleStaticRequest),
-            ('GET', '/', self._handleStaticRequest),
         ]
 
-        if self.sql_handler:
-            routes.append(('GET', '/{tenant}/builds.json',
-                           self._handleSqlRequest))
-            routes.append(('GET', '/{tenant}/builds.html',
-                           self._handleStaticRequest))
+        static_routes = [
+            StaticHandler(self, '/{tenant}/status.html'),
+            StaticHandler(self, '/{tenant}/jobs.html'),
+            StaticHandler(self, '/{tenant}/stream.html'),
+            StaticHandler(self, '/tenants.html', 'index.html'),
+            StaticHandler(self, '/', 'index.html'),
+        ]
+
+        for route in static_routes + self._plugin_routes:
+            routes.append((route.method, route.path, route.handleRequest))
 
         self.log.debug("ZuulWeb starting")
         asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
diff --git a/zuul/web/handler.py b/zuul/web/handler.py
new file mode 100644
index 0000000..43a4695
--- /dev/null
+++ b/zuul/web/handler.py
@@ -0,0 +1,61 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import abc
+import os
+
+from aiohttp import web
+
+STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static')
+
+
+class BaseWebHandler(object, metaclass=abc.ABCMeta):
+
+    def __init__(self, connection, zuul_web, method, path):
+        self.connection = connection
+        self.zuul_web = zuul_web
+        self.method = method
+        self.path = path
+
+    @abc.abstractmethod
+    async def handleRequest(self, request):
+        """Process a web request."""
+
+
+class BaseDriverWebHandler(BaseWebHandler):
+
+    def __init__(self, connection, zuul_web, method, path):
+        super(BaseDriverWebHandler, self).__init__(
+            connection=connection, zuul_web=zuul_web, method=method, path=path)
+        if path.startswith('/'):
+            path = path[1:]
+        self.path = '/connection/{connection}/{path}'.format(
+            connection=self.connection.connection_name,
+            path=path)
+
+
+class StaticHandler(BaseWebHandler):
+
+    def __init__(self, zuul_web, path, file_path=None):
+        super(StaticHandler, self).__init__(None, zuul_web, 'GET', path)
+        self.file_path = file_path or path.split('/')[-1]
+
+    async def handleRequest(self, request):
+        """Process a web request."""
+        headers = {}
+        fp = os.path.join(STATIC_DIR, self.file_path)
+        if self.zuul_web.static_cache_expiry:
+            headers['Cache-Control'] = "public, max-age=%d" % \
+                self.zuul_web.static_cache_expiry
+        return web.FileResponse(fp, headers=headers)
diff --git a/zuul/webapp.py b/zuul/webapp.py
deleted file mode 100644
index b5fdc0e..0000000
--- a/zuul/webapp.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# Copyright 2012 Hewlett-Packard Development Company, L.P.
-# Copyright 2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import copy
-import json
-import logging
-import re
-import threading
-import time
-from paste import httpserver
-import webob
-from webob import dec
-
-from zuul.lib import encryption
-
-"""Zuul main web app.
-
-Zuul supports HTTP requests directly against it for determining the
-change status. These responses are provided as json data structures.
-
-The supported urls are:
-
- - /status: return a complex data structure that represents the entire
-   queue / pipeline structure of the system
- - /status.json (backwards compatibility): same as /status
- - /status/change/X,Y: return status just for gerrit change X,Y
- - /keys/SOURCE/PROJECT.pub: return the public key for PROJECT
-
-When returning status for a single gerrit change you will get an
-array of changes, they will not include the queue structure.
-"""
-
-
-class WebApp(threading.Thread):
-    log = logging.getLogger("zuul.WebApp")
-    change_path_regexp = '/status/change/(.*)$'
-
-    def __init__(self, scheduler, port=8001, cache_expiry=1,
-                 listen_address='0.0.0.0'):
-        threading.Thread.__init__(self)
-        self.scheduler = scheduler
-        self.listen_address = listen_address
-        self.port = port
-        self.cache_expiry = cache_expiry
-        self.cache_time = 0
-        self.cache = {}
-        self.daemon = True
-        self.routes = {}
-        self._init_default_routes()
-        self.server = httpserver.serve(
-            dec.wsgify(self.app), host=self.listen_address, port=self.port,
-            start_loop=False)
-
-    def _init_default_routes(self):
-        self.register_path('/(status\.json|status)$', self.status)
-        self.register_path(self.change_path_regexp, self.change)
-
-    def run(self):
-        self.server.serve_forever()
-
-    def stop(self):
-        self.server.server_close()
-
-    def _changes_by_func(self, func, tenant_name):
-        """Filter changes by a user provided function.
-
-        In order to support arbitrary collection of subsets of changes
-        we provide a low level filtering mechanism that takes a
-        function which applies to changes. The output of this function
-        is a flattened list of those collected changes.
-        """
-        status = []
-        jsonstruct = json.loads(self.cache[tenant_name])
-        for pipeline in jsonstruct['pipelines']:
-            for change_queue in pipeline['change_queues']:
-                for head in change_queue['heads']:
-                    for change in head:
-                        if func(change):
-                            status.append(copy.deepcopy(change))
-        return json.dumps(status)
-
-    def _status_for_change(self, rev, tenant_name):
-        """Return the statuses for a particular change id X,Y."""
-        def func(change):
-            return change['id'] == rev
-        return self._changes_by_func(func, tenant_name)
-
-    def register_path(self, path, handler):
-        path_re = re.compile(path)
-        self.routes[path] = (path_re, handler)
-
-    def unregister_path(self, path):
-        if self.routes.get(path):
-            del self.routes[path]
-
-    def _handle_keys(self, request, path):
-        m = re.match('/keys/(.*?)/(.*?).pub', path)
-        if not m:
-            raise webob.exc.HTTPBadRequest()
-        source_name = m.group(1)
-        project_name = m.group(2)
-        source = self.scheduler.connections.getSource(source_name)
-        if not source:
-            raise webob.exc.HTTPNotFound(
-                detail="Cannot locate a source named %s" % source_name)
-        project = source.getProject(project_name)
-        if not project or not hasattr(project, 'public_key'):
-            raise webob.exc.HTTPNotFound(
-                detail="Cannot locate a project named %s" % project_name)
-
-        pem_public_key = encryption.serialize_rsa_public_key(
-            project.public_key)
-
-        response = webob.Response(body=pem_public_key,
-                                  content_type='text/plain')
-        return response.conditional_response_app
-
-    def app(self, request):
-        # Try registered paths without a tenant_name first
-        path = request.path
-        for path_re, handler in self.routes.values():
-            if path_re.match(path):
-                return handler(path, '', request)
-
-        # Now try with a tenant_name stripped
-        x, tenant_name, path = request.path.split('/', 2)
-        path = '/' + path
-        # Handle keys
-        if path.startswith('/keys'):
-            try:
-                return self._handle_keys(request, path)
-            except Exception as e:
-                self.log.exception("Issue with _handle_keys")
-                raise
-        for path_re, handler in self.routes.values():
-            if path_re.match(path):
-                return handler(path, tenant_name, request)
-        else:
-            raise webob.exc.HTTPNotFound()
-
-    def status(self, path, tenant_name, request):
-        def func():
-            return webob.Response(body=self.cache[tenant_name],
-                                  content_type='application/json',
-                                  charset='utf8')
-        if tenant_name not in self.scheduler.abide.tenants:
-            raise webob.exc.HTTPNotFound()
-        return self._response_with_status_cache(func, tenant_name)
-
-    def change(self, path, tenant_name, request):
-        def func():
-            m = re.match(self.change_path_regexp, path)
-            change_id = m.group(1)
-            status = self._status_for_change(change_id, tenant_name)
-            if status:
-                return webob.Response(body=status,
-                                      content_type='application/json',
-                                      charset='utf8')
-            else:
-                raise webob.exc.HTTPNotFound()
-        return self._response_with_status_cache(func, tenant_name)
-
-    def _refresh_status_cache(self, tenant_name):
-        if (tenant_name not in self.cache or
-            (time.time() - self.cache_time) > self.cache_expiry):
-            try:
-                self.cache[tenant_name] = self.scheduler.formatStatusJSON(
-                    tenant_name)
-                # Call time.time() again because formatting above may take
-                # longer than the cache timeout.
-                self.cache_time = time.time()
-            except Exception:
-                self.log.exception("Exception formatting status:")
-                raise
-
-    def _response_with_status_cache(self, func, tenant_name):
-        self._refresh_status_cache(tenant_name)
-
-        response = func()
-
-        response.headers['Access-Control-Allow-Origin'] = '*'
-
-        response.cache_control.public = True
-        response.cache_control.max_age = self.cache_expiry
-        response.last_modified = self.cache_time
-        response.expires = self.cache_time + self.cache_expiry
-
-        return response.conditional_response_app