Merge "Status: Increase width of change-progress-row-left"
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index 884e25f..4e04b04 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -56,6 +56,10 @@
   Whether to start the internal Gearman server (default: False).
   ``start=true``
 
+**listen_address**
+  IP address or domain name on which to listen (default: all addresses).
+  ``listen_address=127.0.0.1``
+
 **log_config**
   Path to log config file for internal Gearman server.
   ``log_config=/etc/zuul/gearman-logging.yaml``
@@ -462,21 +466,13 @@
     A deprecated alternate spelling of *comment*.  Only one of *comment* or
     *comment_filter* should be used.
 
-    *require-any-approval*
+    *require-approval*
     This may be used for any event.  It requires that a certain kind
     of approval be present for the current patchset of the change (the
     approval could be added by the event in question).  It follows the
     same syntax as the :ref:`"approval" pipeline requirement below
     <pipeline-require-approval>`.
 
-    *require-all-approvals*
-    This takes a list of approvals in the same format as
-    *require-any-approval* but requires all approvals match the rules.
-
-    **require-approval** (depreciated)
-    A deprecated alternate spelling of *require-any-approval*. This will
-    be joined with *require-any-approval* if both are present.
-
   **timer**
     This trigger will run based on a cron-style time specification.
     It will enqueue an event into its pipeline for every project
@@ -523,7 +519,7 @@
 
 .. _pipeline-require-approval:
 
-  **any-approval**
+  **approval**
   This requires that a certain kind of approval be present for the
   current patchset of the change (the approval could be added by the
   event in question).  It takes several sub-parameters, all of which
@@ -557,24 +553,6 @@
     be a single value or a list: ``verified: [1, 2]`` would match
     either a +1 or +2 vote.
 
-    You can also match negative conditions by starting with an
-    exclamation mark (!). This requires the value to be a string.
-    Example: ``verified: '![-1, -2]'``
-
-  This takes a list of approvals in the same format as above. It
-  requires that any approval on a change can meet the specified
-  criteria.
-
-  **all-approvals**
-  This takes a list of approvals in the same format as *any-approval* but
-  requires all approvals match the rules. For example, you can stop any
-  new changes from queueing when there is a negative vote by requiring
-  all approves to not have a -1.
-
-  **approval** (depreciated)
-  A deprecated alternate spelling of *any-approval*. This will be
-  joined with *any-approval* if both are present.
-
   **open**
   A boolean value (``true`` or ``false``) that indicates whether the change
   must be open or closed in order to be enqueued.
diff --git a/tests/fixtures/layout-live-reconfiguration-failed-job.yaml b/tests/fixtures/layout-live-reconfiguration-failed-job.yaml
new file mode 100644
index 0000000..e811af1
--- /dev/null
+++ b/tests/fixtures/layout-live-reconfiguration-failed-job.yaml
@@ -0,0 +1,25 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+jobs:
+  - name: ^.*-merge$
+    failure-message: Unable to merge change
+    hold-following-changes: true
+
+projects:
+  - name: org/project
+    merge-mode: cherry-pick
+    check:
+      - project-merge:
+        - project-test2
+        - project-testfile
diff --git a/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml b/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml
new file mode 100644
index 0000000..ad3f666
--- /dev/null
+++ b/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml
@@ -0,0 +1,62 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: gate
+    manager: DependentPipelineManager
+    failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+jobs:
+  - name: ^.*-merge$
+    failure-message: Unable to merge change
+    hold-following-changes: true
+  - name: project1-project2-integration
+    queue-name: integration
+
+projects:
+  - name: org/project1
+    check:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+    gate:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+
+  - name: org/project2
+    check:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration
+    gate:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration
diff --git a/tests/fixtures/layout-requirement-all.yaml b/tests/fixtures/layout-requirement-all.yaml
deleted file mode 100644
index 968739d..0000000
--- a/tests/fixtures/layout-requirement-all.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-pipelines:
-  - name: pipeline
-    manager: IndependentPipelineManager
-    require:
-      all-approvals:
-        - username: jenkins
-          verified: [1, 2]
-        - verified: "![-1, -2]"
-    trigger:
-      gerrit:
-        - event: comment-added
-    success:
-      gerrit:
-        verified: 1
-    failure:
-      gerrit:
-        verified: -1
-
-  - name: trigger
-    manager: IndependentPipelineManager
-    trigger:
-      gerrit:
-        - event: comment-added
-          require-all-approvals:
-            - username: jenkins
-              verified: [1, 2]
-            - verified: "![-1, -2]"
-    success:
-      gerrit:
-        verified: 1
-    failure:
-      gerrit:
-        verified: -1
-
-projects:
-  - name: org/project1
-    pipeline:
-      - project1-pipeline
-  - name: org/project2
-    trigger:
-      - project2-trigger
diff --git a/tests/fixtures/layout-requirement-any.yaml b/tests/fixtures/layout-requirement-any.yaml
deleted file mode 100644
index 6275d8d..0000000
--- a/tests/fixtures/layout-requirement-any.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-pipelines:
-  - name: pipeline
-    manager: IndependentPipelineManager
-    require:
-      any-approval:
-        - username: jenkins
-          verified: [1, 2]
-        - username: core-reviewer
-          code-review: "![-1, -2]"
-    trigger:
-      gerrit:
-        - event: comment-added
-    success:
-      gerrit:
-        verified: 1
-    failure:
-      gerrit:
-        verified: -1
-
-  - name: trigger
-    manager: IndependentPipelineManager
-    trigger:
-      gerrit:
-        - event: comment-added
-          require-any-approval:
-            - username: jenkins
-              verified: [1, 2]
-            - username: core-reviewer
-              code-review: "![-1, -2]"
-    success:
-      gerrit:
-        verified: 1
-    failure:
-      gerrit:
-        verified: -1
-
-projects:
-  - name: org/project1
-    pipeline:
-      - project1-pipeline
-  - name: org/project2
-    trigger:
-      - project2-trigger
diff --git a/tests/fixtures/layout-requirement-email.yaml b/tests/fixtures/layout-requirement-email.yaml
index dadcd6c..4bfb733 100644
--- a/tests/fixtures/layout-requirement-email.yaml
+++ b/tests/fixtures/layout-requirement-email.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - email: jenkins@example.com
     trigger:
       gerrit:
@@ -19,7 +19,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - email: jenkins@example.com
     success:
       gerrit:
diff --git a/tests/fixtures/layout-requirement-negative-username.yaml b/tests/fixtures/layout-requirement-negative-username.yaml
deleted file mode 100644
index f542b86..0000000
--- a/tests/fixtures/layout-requirement-negative-username.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-pipelines:
-  - name: pipeline
-    manager: IndependentPipelineManager
-    require:
-      all-approvals:
-        - username: '!jenkins'
-    trigger:
-      gerrit:
-        - event: comment-added
-    success:
-      gerrit:
-        verified: 1
-    failure:
-      gerrit:
-        verified: -1
-
-  - name: trigger
-    manager: IndependentPipelineManager
-    trigger:
-      gerrit:
-        - event: comment-added
-          require-all-approvals:
-            - username: '!jenkins'
-    success:
-      gerrit:
-        verified: 1
-    failure:
-      gerrit:
-        verified: -1
-
-projects:
-  - name: org/project1
-    pipeline:
-      - project1-pipeline
-  - name: org/project2
-    trigger:
-      - project2-trigger
\ No newline at end of file
diff --git a/tests/fixtures/layout-requirement-newer-than.yaml b/tests/fixtures/layout-requirement-newer-than.yaml
index f723c79..b6beb35 100644
--- a/tests/fixtures/layout-requirement-newer-than.yaml
+++ b/tests/fixtures/layout-requirement-newer-than.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - username: jenkins
           newer-than: 48h
     trigger:
@@ -20,7 +20,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - username: jenkins
               newer-than: 48h
     success:
diff --git a/tests/fixtures/layout-requirement-older-than.yaml b/tests/fixtures/layout-requirement-older-than.yaml
index 0e011cc..2edf9df 100644
--- a/tests/fixtures/layout-requirement-older-than.yaml
+++ b/tests/fixtures/layout-requirement-older-than.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - username: jenkins
           older-than: 48h
     trigger:
@@ -20,7 +20,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - username: jenkins
               older-than: 48h
     success:
diff --git a/tests/fixtures/layout-requirement-username.yaml b/tests/fixtures/layout-requirement-username.yaml
index 8520179..7a549f0 100644
--- a/tests/fixtures/layout-requirement-username.yaml
+++ b/tests/fixtures/layout-requirement-username.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - username: jenkins
     trigger:
       gerrit:
@@ -19,7 +19,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - username: jenkins
     success:
       gerrit:
diff --git a/tests/fixtures/layout-requirement-vote.yaml b/tests/fixtures/layout-requirement-vote.yaml
index 6736e98..7ccadff 100644
--- a/tests/fixtures/layout-requirement-vote.yaml
+++ b/tests/fixtures/layout-requirement-vote.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - username: jenkins
           verified: 1
     trigger:
@@ -20,7 +20,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - username: jenkins
               verified: 1
     success:
diff --git a/tests/fixtures/layout-requirement-vote1.yaml b/tests/fixtures/layout-requirement-vote1.yaml
index 6736e98..7ccadff 100644
--- a/tests/fixtures/layout-requirement-vote1.yaml
+++ b/tests/fixtures/layout-requirement-vote1.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - username: jenkins
           verified: 1
     trigger:
@@ -20,7 +20,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - username: jenkins
               verified: 1
     success:
diff --git a/tests/fixtures/layout-requirement-vote2.yaml b/tests/fixtures/layout-requirement-vote2.yaml
index a6cd6a3..33d84d1 100644
--- a/tests/fixtures/layout-requirement-vote2.yaml
+++ b/tests/fixtures/layout-requirement-vote2.yaml
@@ -2,7 +2,7 @@
   - name: pipeline
     manager: IndependentPipelineManager
     require:
-      any-approval:
+      approval:
         - username: jenkins
           verified: [1, 2]
     trigger:
@@ -20,7 +20,7 @@
     trigger:
       gerrit:
         - event: comment-added
-          require-any-approval:
+          require-approval:
             - username: jenkins
               verified: [1, 2]
     success:
diff --git a/tests/test_requirements.py b/tests/test_requirements.py
index 52e3973..4316925 100644
--- a/tests/test_requirements.py
+++ b/tests/test_requirements.py
@@ -323,131 +323,3 @@
         self.fake_gerrit.addEvent(B.addApproval('CRVW', 2))
         self.waitUntilSettled()
         self.assertEqual(len(self.history), 1)
-
-    def test_pipeline_require_negative_username(self):
-        "Test negative pipeline requirement: no comment from jenkins"
-        return self._test_require_negative_username('org/project1',
-                                                    'project1-pipeline')
-
-    def test_trigger_require_negative_username(self):
-        "Test negative trigger requirement: no comment from jenkins"
-        return self._test_require_negative_username('org/project2',
-                                                    'project2-trigger')
-
-    def _test_require_negative_username(self, project, job):
-        "Test negative username's match"
-        # Should only trigger if Jenkins hasn't voted.
-        self.config.set(
-            'zuul', 'layout_config',
-            'tests/fixtures/layout-requirement-negative-username.yaml')
-        self.sched.reconfigure(self.config)
-        self.registerJobs()
-
-        # add in a change with no comments
-        A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 0)
-
-        # add in a comment that will trigger
-        self.fake_gerrit.addEvent(A.addApproval('CRVW', 1,
-                                                username='reviewer'))
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 1)
-        self.assertEqual(self.history[0].name, job)
-
-        # add in a comment from jenkins user which shouldn't trigger
-        self.fake_gerrit.addEvent(A.addApproval('VRFY', 1, username='jenkins'))
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 1)
-
-        # Check future reviews also won't trigger as a 'jenkins' user has
-        # commented previously
-        self.fake_gerrit.addEvent(A.addApproval('CRVW', 1,
-                                                username='reviewer'))
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 1)
-
-    def test_pipeline_require_any(self):
-        "Test pipeline requirement: any requirement passes"
-        return self._test_require_any('org/project1', 'project1-pipeline')
-
-    def test_trigger_require_any(self):
-        "Test trigger requirement: any requirement passes"
-        return self._test_require_any('org/project2', 'project2-trigger')
-
-    def _test_require_any(self, project, job):
-        "Test any of the given requirements are matched"
-        self.config.set(
-            'zuul', 'layout_config',
-            'tests/fixtures/layout-requirement-any.yaml')
-        self.sched.reconfigure(self.config)
-        self.registerJobs()
-
-        A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
-        # A comment event that we will keep submitting to trigger
-        comment = A.addApproval('CRVW', 1, username='nobody')
-        self.fake_gerrit.addEvent(comment)
-        self.waitUntilSettled()
-        # No approval from Jenkins so should not be enqueued
-        self.assertEqual(len(self.history), 0)
-
-        # A +1 from jenkins should allow it to be enqueued
-        A.addApproval('VRFY', 1, username='jenkins')
-        self.fake_gerrit.addEvent(comment)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 1)
-        self.assertEqual(self.history[0].name, job)
-
-        # A non-negative from a non-core should not queue
-        B = self.fake_gerrit.addFakeChange(project, 'master', 'B')
-        # A comment event that we will keep submitting to trigger
-        comment = B.addApproval('CRVW', 1, username='nobody')
-        self.fake_gerrit.addEvent(comment)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 1)
-
-        # A non-negative from a core member should queue
-        B.addApproval('CRVW', 2, username='core-reviewer')
-        self.fake_gerrit.addEvent(comment)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 2)
-        self.assertEqual(self.history[1].name, job)
-
-    def test_pipeline_require_all(self):
-        "Test pipeline requirement: all requirements pass"
-        return self._test_require_all('org/project1', 'project1-pipeline')
-
-    def test_trigger_require_all(self):
-        "Test trigger requirement: all requirements pass"
-        return self._test_require_all('org/project2', 'project2-trigger')
-
-    def _test_require_all(self, project, job):
-        "Test all of the given requirements are matched"
-        self.config.set(
-            'zuul', 'layout_config',
-            'tests/fixtures/layout-requirement-all.yaml')
-        self.sched.reconfigure(self.config)
-        self.registerJobs()
-
-        A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 0)
-
-        # A +2 from 'nobody' only satisfies the non-negative requirement,
-        # not the requirement to be from 'jenkins'
-        comment = A.addApproval('VRFY', 1, username='nobody')
-        self.fake_gerrit.addEvent(comment)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 0)
-
-        B = self.fake_gerrit.addFakeChange(project, 'master', 'A')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 0)
-
-        # A +2 from Jenkins satisfies both the user condition and the
-        # non-negative condition
-        comment = B.addApproval('VRFY', 2, username='jenkins')
-        self.fake_gerrit.addEvent(comment)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.history), 1)
-        self.assertEqual(self.history[0].name, job)
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 6c82748..61a2d09 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2280,7 +2280,6 @@
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
         A.addPatchset(['conflict'])
         A.addApproval('CRVW', 2)
-        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
 
         # This change will be in merge conflict.  During the
         # reconfiguration, we will add a job.  We want to make sure
@@ -2288,6 +2287,8 @@
         B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
         B.addPatchset(['conflict'])
         B.addApproval('CRVW', 2)
+
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
         self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
 
         self.waitUntilSettled()
@@ -2324,7 +2325,7 @@
                          'SUCCESS')
         self.assertEqual(len(self.history), 4)
 
-    def test_live_reconfiguration_failed_job(self):
+    def test_live_reconfiguration_failed_root(self):
         # An extrapolation of test_live_reconfiguration_merge_conflict
         # that tests a job added to a job tree with a failed root does
         # not run.
@@ -2386,6 +2387,113 @@
         self.assertEqual(self.history[4].result, 'SUCCESS')
         self.assertEqual(len(self.history), 5)
 
+    def test_live_reconfiguration_failed_job(self):
+        # Test that a change with a removed failing job does not
+        # disrupt reconfiguration.  If a change has a failed job and
+        # that job is removed during a reconfiguration, we observed a
+        # bug where the code to re-set build statuses would run on
+        # that build and raise an exception because the job no longer
+        # existed.
+        self.worker.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+
+        # This change will fail and later be removed by the reconfiguration.
+        self.worker.addFailTest('project-test1', A)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.worker.release('.*-merge')
+        self.waitUntilSettled()
+        self.worker.release('project-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 0)
+
+        self.assertEqual(self.getJobFromHistory('project-merge').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project-test1').result,
+                         'FAILURE')
+        self.assertEqual(len(self.history), 2)
+
+        # Remove the test1 job.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-live-'
+                        'reconfiguration-failed-job.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(self.getJobFromHistory('project-test2').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project-testfile').result,
+                         'SUCCESS')
+        self.assertEqual(len(self.history), 4)
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 1)
+        self.assertIn('Build succeeded', A.messages[0])
+        # Ensure the removed job was not included in the report.
+        self.assertNotIn('project-test1', A.messages[0])
+
+    def test_live_reconfiguration_shared_queue(self):
+        # Test that a change with a failing job which was removed from
+        # this project but otherwise still exists in the system does
+        # not disrupt reconfiguration.
+
+        self.worker.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+
+        self.worker.addFailTest('project1-project2-integration', A)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.worker.release('.*-merge')
+        self.waitUntilSettled()
+        self.worker.release('project1-project2-integration')
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 0)
+
+        self.assertEqual(self.getJobFromHistory('project1-merge').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory(
+            'project1-project2-integration').result, 'FAILURE')
+        self.assertEqual(len(self.history), 2)
+
+        # Remove the integration job.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-live-'
+                        'reconfiguration-shared-queue.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(self.getJobFromHistory('project1-merge').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project1-test1').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project1-test2').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory(
+            'project1-project2-integration').result, 'FAILURE')
+        self.assertEqual(len(self.history), 4)
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 1)
+        self.assertIn('Build succeeded', A.messages[0])
+        # Ensure the removed job was not included in the report.
+        self.assertNotIn('project1-project2-integration', A.messages[0])
+
     def test_live_reconfiguration_functions(self):
         "Test live reconfiguration with a custom function"
         self.worker.registerFunction('build:node-project-test1:debian')
@@ -2628,7 +2736,7 @@
         self.worker.release('.*')
         self.waitUntilSettled()
 
-    def test_client_enqueue(self):
+    def test_client_enqueue_change(self):
         "Test that the RPC client can enqueue a change"
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
         A.addApproval('CRVW', 2)
@@ -2651,6 +2759,24 @@
         self.assertEqual(A.reported, 2)
         self.assertEqual(r, True)
 
+    def test_client_enqueue_ref(self):
+        "Test that the RPC client can enqueue a ref"
+
+        client = zuul.rpcclient.RPCClient('127.0.0.1',
+                                          self.gearman_server.port)
+        r = client.enqueue_ref(
+            pipeline='post',
+            project='org/project',
+            trigger='gerrit',
+            ref='master',
+            oldrev='90f173846e3af9154517b88543ffbd1691f31366',
+            newrev='d479a0bfcb34da57a31adb2a595c0cf687812543')
+        self.waitUntilSettled()
+        job_names = [x.name for x in self.history]
+        self.assertEqual(len(self.history), 1)
+        self.assertIn('project-post', job_names)
+        self.assertEqual(r, True)
+
     def test_client_enqueue_negative(self):
         "Test that the RPC client returns errors"
         client = zuul.rpcclient.RPCClient('127.0.0.1',
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index bc2c152..6e14ff5 100644
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -56,6 +56,24 @@
                                  required=True)
         cmd_enqueue.set_defaults(func=self.enqueue)
 
+        cmd_enqueue = subparsers.add_parser('enqueue-ref',
+                                            help='enqueue a ref')
+        cmd_enqueue.add_argument('--trigger', help='trigger name',
+                                 required=True)
+        cmd_enqueue.add_argument('--pipeline', help='pipeline name',
+                                 required=True)
+        cmd_enqueue.add_argument('--project', help='project name',
+                                 required=True)
+        cmd_enqueue.add_argument('--ref', help='ref name',
+                                 required=True)
+        cmd_enqueue.add_argument(
+            '--oldrev', help='old revision',
+            default='0000000000000000000000000000000000000000')
+        cmd_enqueue.add_argument(
+            '--newrev', help='new revision',
+            default='0000000000000000000000000000000000000000')
+        cmd_enqueue.set_defaults(func=self.enqueue_ref)
+
         cmd_promote = subparsers.add_parser('promote',
                                             help='promote one or more changes')
         cmd_promote.add_argument('--pipeline', help='pipeline name',
@@ -82,6 +100,9 @@
         show_running_jobs.set_defaults(func=self.show_running_jobs)
 
         self.args = parser.parse_args()
+        if self.args.func == self.enqueue_ref:
+            if self.args.oldrev == self.args.newrev:
+                parser.error("The old and new revisions must not be the same.")
 
     def setup_logging(self):
         """Client logging does not rely on conf file"""
@@ -112,6 +133,16 @@
                            change=self.args.change)
         return r
 
+    def enqueue_ref(self):
+        client = zuul.rpcclient.RPCClient(self.server, self.port)
+        r = client.enqueue_ref(pipeline=self.args.pipeline,
+                               project=self.args.project,
+                               trigger=self.args.trigger,
+                               ref=self.args.ref,
+                               oldrev=self.args.oldrev,
+                               newrev=self.args.newrev)
+        return r
+
     def promote(self):
         client = zuul.rpcclient.RPCClient(self.server, self.port)
         r = client.promote(pipeline=self.args.pipeline,
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index 5bbe3a0..187b6b0 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -88,17 +88,14 @@
             )
 
         args = parser.parse_args()
+        # Validate ZUUL_* arguments. If ref is provided then URL is required.
+        zuul_args = [zuul_opt for zuul_opt, val in vars(args).items()
+                     if zuul_opt.startswith('zuul') and val is not None]
+        if 'zuul_ref' in zuul_args and 'zuul_url' not in zuul_args:
+            parser.error("Specifying a Zuul ref requires a Zuul url. "
+                         "Define Zuul arguments either via environment "
+                         "variables or using options above.")
 
-        # Validate ZUUL_* arguments. If any ZUUL_* argument is set they
-        # must all be set, otherwise fallback to defaults.
-        zuul_missing = [zuul_opt for zuul_opt, val in vars(args).items()
-                        if zuul_opt.startswith('zuul') and val is None]
-        if (len(zuul_missing) > 0 and
-            len(zuul_missing) < len(ZUUL_ENV_SUFFIXES)):
-            parser.error(("Some Zuul parameters are not set:\n\t%s\n"
-                          "Define them either via environment variables or "
-                          "using options above." %
-                          "\n\t".join(sorted(zuul_missing))))
         self.args = args
 
     def setup_logging(self, color=False, verbose=False):
diff --git a/zuul/cmd/server.py b/zuul/cmd/server.py
index a3a23ad..2d99a1f 100755
--- a/zuul/cmd/server.py
+++ b/zuul/cmd/server.py
@@ -62,7 +62,10 @@
         signal.signal(signal.SIGHUP, signal.SIG_IGN)
         self.read_config()
         self.setup_logging('zuul', 'log_config')
-        self.sched.reconfigure(self.config)
+        try:
+            self.sched.reconfigure(self.config)
+        except Exception:
+            self.log.exception("Reconfiguration failed:")
         signal.signal(signal.SIGHUP, self.reconfigure_handler)
 
     def exit_handler(self, signum, frame):
@@ -118,8 +121,12 @@
             import gear
             statsd_host = os.environ.get('STATSD_HOST')
             statsd_port = int(os.environ.get('STATSD_PORT', 8125))
+            if self.config.has_option('gearman_server', 'listen_address'):
+                host = self.config.get('gearman_server', 'listen_address')
+            else:
+                host = None
             gear.Server(4730,
-                        host=self.config.get('gearman', 'server'),
+                        host=host,
                         statsd_host=statsd_host,
                         statsd_port=statsd_port,
                         statsd_prefix='zuul.geard')
diff --git a/zuul/launcher/gearman.py b/zuul/launcher/gearman.py
index 57ac5ca..828e2a9 100644
--- a/zuul/launcher/gearman.py
+++ b/zuul/launcher/gearman.py
@@ -342,8 +342,7 @@
         build.parameters = params
 
         if job.name == 'noop':
-            build.result = 'SUCCESS'
-            self.sched.onBuildCompleted(build)
+            self.sched.onBuildCompleted(build, 'SUCCESS')
             return build
 
         gearman_job = gear.Job(name, json.dumps(params),
@@ -431,8 +430,7 @@
                     build.retry = True
                 self.log.info("Build %s complete, result %s" %
                               (job, result))
-                build.result = result
-                self.sched.onBuildCompleted(build)
+                self.sched.onBuildCompleted(build, result)
             # The test suite expects the build to be removed from the
             # internal dict after it's added to the report queue.
             del self.builds[job.unique]
diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py
index 1569fa9..88d10e2 100644
--- a/zuul/layoutvalidator.py
+++ b/zuul/layoutvalidator.py
@@ -62,8 +62,6 @@
                       'branch': toList(str),
                       'ref': toList(str),
                       'approval': toList(variable_dict),
-                      'require-all-approvals': toList(require_approval),
-                      'require-any-approval': toList(require_approval),
                       'require-approval': toList(require_approval),
                       }
 
@@ -87,9 +85,7 @@
                                },
                       }
 
-    require = {'all-approvals': toList(require_approval),
-               'any-approval': toList(require_approval),
-               'approval': toList(require_approval),
+    require = {'approval': toList(require_approval),
                'open': bool,
                'current-patchset': bool,
                'status': toList(str)}
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index d697648..0ac7f0f 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -138,8 +138,11 @@
         if project in self.project_branches:
             indicated_branch = self.project_branches[project]
 
-        override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
-                                   self.zuul_ref)
+        if indicated_branch:
+            override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
+                                       self.zuul_ref)
+        else:
+            override_zuul_ref = None
 
         if indicated_branch and repo.hasBranch(indicated_branch):
             self.log.info("upstream repo has branch %s", indicated_branch)
@@ -150,14 +153,18 @@
             # FIXME should be origin HEAD branch which might not be 'master'
             fallback_branch = 'master'
 
-        fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
-                                   self.zuul_ref)
+        if self.zuul_branch:
+            fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+                                       self.zuul_ref)
+        else:
+            fallback_zuul_ref = None
 
         # If we have a non empty zuul_ref to use, use it. Otherwise we fall
         # back to checking out the branch.
         if ((override_zuul_ref and
             self.fetchFromZuul(repo, project, override_zuul_ref)) or
-            (fallback_zuul_ref != override_zuul_ref and
+            (fallback_zuul_ref and
+             fallback_zuul_ref != override_zuul_ref and
             self.fetchFromZuul(repo, project, fallback_zuul_ref))):
             # Work around a bug in GitPython which can not parse FETCH_HEAD
             gitcmd = git.Git(dest)
@@ -169,9 +176,9 @@
             # Checkout branch
             self.log.info("Falling back to branch %s", fallback_branch)
             try:
-                repo.checkout('remotes/origin/%s' % fallback_branch)
+                commit = repo.checkout('remotes/origin/%s' % fallback_branch)
             except (ValueError, GitCommandError):
                 self.log.exception("Fallback branch not found: %s",
                                    fallback_branch)
-            self.log.info("Prepared %s repo with branch %s",
-                          project, fallback_branch)
+            self.log.info("Prepared %s repo with branch %s at commit %s",
+                          project, fallback_branch, commit)
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index f571ad0..c84d042 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -86,9 +86,9 @@
         return repo
 
     def reset(self):
-        repo = self.createRepoObject()
         self.log.debug("Resetting repository %s" % self.local_path)
         self.update()
+        repo = self.createRepoObject()
         origin = repo.remotes.origin
         for ref in origin.refs:
             if ref.remote_head == 'HEAD':
@@ -130,6 +130,7 @@
         self.log.debug("Checking out %s" % ref)
         repo.head.reference = ref
         reset_repo_to_head(repo)
+        return repo.head.commit
 
     def cherryPick(self, ref):
         repo = self.createRepoObject()
diff --git a/zuul/model.py b/zuul/model.py
index 6648774..4b907c3 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -12,10 +12,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import ast
 import copy
 import re
-import six
 import time
 from uuid import uuid4
 import extras
@@ -1027,127 +1025,68 @@
 
 
 class BaseFilter(object):
-    def __init__(self, required_any_approval=[], required_all_approvals=[]):
-        self._required_any_approval = copy.deepcopy(required_any_approval)
-        self.required_any_approval = self._tidy_approvals(
-            required_any_approval)
-        self._required_all_approvals = copy.deepcopy(required_all_approvals)
-        self.required_all_approvals = self._tidy_approvals(
-            required_all_approvals)
+    def __init__(self, required_approvals=[]):
+        self._required_approvals = copy.deepcopy(required_approvals)
+        self.required_approvals = required_approvals
 
-    def _tidy_approvals(self, approvals):
-        for a in approvals:
+        for a in self.required_approvals:
             for k, v in a.items():
                 if k == 'username':
                     pass
                 elif k in ['email', 'email-filter']:
-                    a['email'] = v
+                    a['email'] = re.compile(v)
                 elif k == 'newer-than':
-                    a[k] = v
+                    a[k] = time_to_seconds(v)
                 elif k == 'older-than':
-                    a[k] = v
+                    a[k] = time_to_seconds(v)
+                else:
+                    if not isinstance(v, list):
+                        a[k] = [v]
             if 'email-filter' in a:
                 del a['email-filter']
-        return approvals
-
-    def _match_approval_required_approval(self, rapproval, approval):
-        # Check if the required approval and approval match
-        if 'description' not in approval:
-            return False
-        now = time.time()
-        found_approval = True
-        by = approval.get('by', {})
-        for k, v in rapproval.items():
-            negative_match = False
-            item_match = True
-            if isinstance(v, six.string_types) and v[0] == '!':
-                v = v[1:].strip()
-                item_match = False
-                negative_match = True
-
-            if k == 'username':
-                if (by.get('username', '') != v):
-                        item_match = negative_match
-            elif k == 'email':
-                v = re.compile(v)
-                if (not v.search(by.get('email', ''))):
-                        item_match = negative_match
-            elif k == 'newer-than':
-                t = now - time_to_seconds(v)
-                if (approval['grantedOn'] < t):
-                        item_match = negative_match
-            elif k == 'older-than':
-                t = now - time_to_seconds(v)
-                if (approval['grantedOn'] >= t):
-                    item_match = negative_match
-            else:
-                if isinstance(v, six.string_types):
-                    v = ast.literal_eval(v)
-                if not isinstance(v, list):
-                    v = [v]
-                if (normalizeCategory(approval['description']) != k or
-                        int(approval['value']) not in v):
-                    item_match = negative_match
-            if not item_match:
-                found_approval = False
-        return found_approval
 
     def matchesRequiredApprovals(self, change):
-        if (self.required_any_approval and not change.approvals
-                or self.required_all_approvals and not change.approvals):
-            # A change with no approvals can not match
-            return False
-
-        # TODO(jhesketh): If we wanted to optimise this slightly we could
-        # analyse both the ANY and ALL filters by looping over the approvals
-        # on the change and keeping track of what we have checked rather than
-        # needing to loop on the change approvals twice
-        return (self.matchesRequiredAnyApproval(change) and
-                self.matchesRequiredAllApprovals(change))
-
-    def matchesRequiredAnyApproval(self, change):
-        # Check if any approvals match the any requirements
-        if not self.required_any_approval:
-            # No approval required, so we must match
-            return True
-
-        for rapproval in self.required_any_approval:
+        now = time.time()
+        for rapproval in self.required_approvals:
             matches_approval = False
             for approval in change.approvals:
-                matches_approval = self._match_approval_required_approval(
-                    rapproval, approval)
-                if matches_approval:
-                    # We have a matching approval so this requirement is
-                    # fulfilled
-                    return True
-        return False
-
-    def matchesRequiredAllApprovals(self, change):
-        # Check that /all/ of the approvals match the requirements
-        if not self.required_all_approvals:
-            # No approvals required, so we must match
-            return True
-
-        for rapproval in self.required_all_approvals:
-            for approval in change.approvals:
-                matches_approval = self._match_approval_required_approval(
-                    rapproval, approval)
-                if not matches_approval:
-                    # We have an approval that doesn't match so this
-                    # requirement can't be fulfilled
-                    return False
-        # We must have matched everything
+                if 'description' not in approval:
+                    continue
+                found_approval = True
+                by = approval.get('by', {})
+                for k, v in rapproval.items():
+                    if k == 'username':
+                        if (by.get('username', '') != v):
+                            found_approval = False
+                    elif k == 'email':
+                        if (not v.search(by.get('email', ''))):
+                            found_approval = False
+                    elif k == 'newer-than':
+                        t = now - v
+                        if (approval['grantedOn'] < t):
+                            found_approval = False
+                    elif k == 'older-than':
+                        t = now - v
+                        if (approval['grantedOn'] >= t):
+                            found_approval = False
+                    else:
+                        if (normalizeCategory(approval['description']) != k or
+                            int(approval['value']) not in v):
+                            found_approval = False
+                if found_approval:
+                    matches_approval = True
+                    break
+            if not matches_approval:
+                return False
         return True
 
 
 class EventFilter(BaseFilter):
     def __init__(self, trigger, types=[], branches=[], refs=[],
                  event_approvals={}, comments=[], emails=[], usernames=[],
-                 timespecs=[], required_any_approval=[],
-                 required_all_approvals=[], pipelines=[]):
+                 timespecs=[], required_approvals=[], pipelines=[]):
         super(EventFilter, self).__init__(
-            required_any_approval=required_any_approval,
-            required_all_approvals=required_all_approvals)
+            required_approvals=required_approvals)
         self.trigger = trigger
         self._types = types
         self._branches = branches
@@ -1180,12 +1119,9 @@
         if self.event_approvals:
             ret += ' event_approvals: %s' % ', '.join(
                 ['%s:%s' % a for a in self.event_approvals.items()])
-        if self.required_any_approval:
-            ret += ' required_any_approval: %s' % ', '.join(
-                ['%s' % a for a in self._required_any_approval])
-        if self.required_all_approvals:
-            ret += ' required_all_approvals: %s' % ', '.join(
-                ['%s' % a for a in self._required_all_approvals])
+        if self.required_approvals:
+            ret += ' required_approvals: %s' % ', '.join(
+                ['%s' % a for a in self._required_approvals])
         if self._comments:
             ret += ' comments: %s' % ', '.join(self._comments)
         if self._emails:
@@ -1274,6 +1210,10 @@
             if not matches_approval:
                 return False
 
+        if self.required_approvals and not change.approvals:
+            # A change with no approvals can not match
+            return False
+
         # required approvals are ANDed
         if not self.matchesRequiredApprovals(change):
             return False
@@ -1291,11 +1231,9 @@
 
 class ChangeishFilter(BaseFilter):
     def __init__(self, open=None, current_patchset=None,
-                 statuses=[], required_any_approval=[],
-                 required_all_approvals=[]):
+                 statuses=[], required_approvals=[]):
         super(ChangeishFilter, self).__init__(
-            required_any_approval=required_any_approval,
-            required_all_approvals=required_all_approvals)
+            required_approvals=required_approvals)
         self.open = open
         self.current_patchset = current_patchset
         self.statuses = statuses
@@ -1309,12 +1247,8 @@
             ret += ' current-patchset: %s' % self.current_patchset
         if self.statuses:
             ret += ' statuses: %s' % ', '.join(self.statuses)
-        if self.required_any_approval:
-            ret += (' required_any_approval: %s' %
-                    str(self.required_any_approval))
-        if self.required_all_approvals:
-            ret += (' required_all_approvals: %s' %
-                    str(self.required_all_approvals))
+        if self.required_approvals:
+            ret += ' required_approvals: %s' % str(self.required_approvals)
         ret += '>'
 
         return ret
@@ -1332,6 +1266,10 @@
             if change.status not in self.statuses:
                 return False
 
+        if self.required_approvals and not change.approvals:
+            # A change with no approvals can not match
+            return False
+
         # required approvals are ANDed
         if not self.matchesRequiredApprovals(change):
             return False
diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py
index f43c3b9..609f636 100644
--- a/zuul/rpcclient.py
+++ b/zuul/rpcclient.py
@@ -56,6 +56,16 @@
                 }
         return not self.submitJob('zuul:enqueue', data).failure
 
+    def enqueue_ref(self, pipeline, project, trigger, ref, oldrev, newrev):
+        data = {'pipeline': pipeline,
+                'project': project,
+                'trigger': trigger,
+                'ref': ref,
+                'oldrev': oldrev,
+                'newrev': newrev,
+                }
+        return not self.submitJob('zuul:enqueue_ref', data).failure
+
     def promote(self, pipeline, change_ids):
         data = {'pipeline': pipeline,
                 'change_ids': change_ids,
diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py
index 05b8d03..d54da9f 100644
--- a/zuul/rpclistener.py
+++ b/zuul/rpclistener.py
@@ -48,6 +48,7 @@
 
     def register(self):
         self.worker.registerFunction("zuul:enqueue")
+        self.worker.registerFunction("zuul:enqueue_ref")
         self.worker.registerFunction("zuul:promote")
         self.worker.registerFunction("zuul:get_running_jobs")
 
@@ -83,7 +84,7 @@
             except Exception:
                 self.log.exception("Exception while getting job")
 
-    def handle_enqueue(self, job):
+    def _common_enqueue(self, job):
         args = json.loads(job.arguments)
         event = model.TriggerEvent()
         errors = ''
@@ -106,6 +107,11 @@
         else:
             errors += 'Invalid pipeline: %s\n' % (args['pipeline'],)
 
+        return (args, event, errors, pipeline, project)
+
+    def handle_enqueue(self, job):
+        (args, event, errors, pipeline, project) = self._common_enqueue(job)
+
         if not errors:
             event.change_number, event.patch_number = args['change'].split(',')
             try:
@@ -119,6 +125,20 @@
             self.sched.enqueue(event)
             job.sendWorkComplete()
 
+    def handle_enqueue_ref(self, job):
+        (args, event, errors, pipeline, project) = self._common_enqueue(job)
+
+        if not errors:
+            event.ref = args['ref']
+            event.oldrev = args['oldrev']
+            event.newrev = args['newrev']
+
+        if errors:
+            job.sendWorkException(errors.encode('utf8'))
+        else:
+            self.sched.enqueue(event)
+            job.sendWorkComplete()
+
     def handle_promote(self, job):
         args = json.loads(job.arguments)
         pipeline_name = args['pipeline']
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a3d9978..3562c68 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -327,10 +327,7 @@
                     open=require.get('open'),
                     current_patchset=require.get('current-patchset'),
                     statuses=toList(require.get('status')),
-                    required_any_approval=(toList(require.get('any-approval'))
-                                           + toList(require.get('approval'))),
-                    required_all_approvals=toList(require.get('all-approvals'))
-                )
+                    required_approvals=toList(require.get('approval')))
                 manager.changeish_filters.append(f)
 
             # TODO: move this into triggers (may require pluggable
@@ -360,13 +357,9 @@
                         comments=comments,
                         emails=emails,
                         usernames=usernames,
-                        required_any_approval=(
-                            toList(trigger.get('require-any-approval'))
-                            + toList(trigger.get('require-approval'))
-                        ),
-                        required_all_approvals=toList(
-                            trigger.get('require-all-approvals')
-                        ),
+                        required_approvals=toList(
+                            trigger.get('require-approval')
+                        )
                     )
                     manager.event_filters.append(f)
             if 'timer' in conf_pipeline['trigger']:
@@ -381,13 +374,9 @@
                         trigger=self.triggers['zuul'],
                         types=toList(trigger['event']),
                         pipelines=toList(trigger.get('pipeline')),
-                        required_any_approval=(
-                            toList(trigger.get('require-any-approval'))
-                            + toList(trigger.get('require-approval'))
-                        ),
-                        required_all_approvals=toList(
-                            trigger.get('require-all-approvals')
-                        ),
+                        required_approvals=toList(
+                            trigger.get('require-approval')
+                        )
                     )
                     manager.event_filters.append(f)
 
@@ -550,9 +539,15 @@
         self.wake_event.set()
         self.log.debug("Done adding start event for build: %s" % build)
 
-    def onBuildCompleted(self, build):
-        self.log.debug("Adding complete event for build: %s" % build)
+    def onBuildCompleted(self, build, result):
+        self.log.debug("Adding complete event for build: %s result: %s" % (
+            build, result))
         build.end_time = time.time()
+        # Note, as soon as the result is set, other threads may act
+        # upon this, even though the event hasn't been fully
+        # processed.  Ensure that any other data from the event (eg,
+        # timing) is recorded before setting the result.
+        build.result = result
         try:
             if statsd and build.pipeline:
                 jobname = build.job.name.replace('.', '_')
@@ -684,7 +679,7 @@
                     continue
                 self.log.debug("Re-enqueueing changes for pipeline %s" % name)
                 items_to_remove = []
-                builds_to_remove = []
+                builds_to_cancel = []
                 last_head = None
                 for shared_queue in old_pipeline.queues:
                     for item in shared_queue.queue:
@@ -703,19 +698,21 @@
                             items_to_remove.append(item)
                             continue
                         item.change.project = project
+                        item_jobs = new_pipeline.getJobs(item)
                         for build in item.current_build_set.getBuilds():
                             job = layout.jobs.get(build.job.name)
-                            if job:
+                            if job and job in item_jobs:
                                 build.job = job
                             else:
-                                builds_to_remove.append(build)
+                                item.removeBuild(build)
+                                builds_to_cancel.append(build)
                         if not new_pipeline.manager.reEnqueueItem(item,
                                                                   last_head):
                             items_to_remove.append(item)
                 for item in items_to_remove:
                     for build in item.current_build_set.getBuilds():
-                        builds_to_remove.append(build)
-                for build in builds_to_remove:
+                        builds_to_cancel.append(build)
+                for build in builds_to_cancel:
                     self.log.warning(
                         "Canceling build %s during reconfiguration" % (build,))
                     try:
@@ -1367,6 +1364,7 @@
                                    "for change %s" % (build, item.change))
             build.result = 'CANCELED'
             canceled = True
+        self.updateBuildDescriptions(old_build_set)
         for item_behind in item.items_behind:
             self.log.debug("Canceling jobs for change %s, behind change %s" %
                            (item_behind.change, item.change))
@@ -1490,7 +1488,6 @@
 
     def onBuildStarted(self, build):
         self.log.debug("Build %s started" % build)
-        self.updateBuildDescriptions(build.build_set)
         return True
 
     def onBuildCompleted(self, build):
@@ -1500,7 +1497,6 @@
         self.pipeline.setResult(item, build)
         self.log.debug("Item %s status is now:\n %s" %
                        (item, item.formatStatus()))
-        self.updateBuildDescriptions(build.build_set)
         return True
 
     def onMergeCompleted(self, event):