Fix bug with shared jobs in reconfiguration

This is a variant on the bug fixed in
I2d5cc8f4c5b73d6a2b184cd21016943b8c08949d.
This corrects the case where a job is removed from a project and
there is a current failed build of that job for that project in the
queue, yet, unlike the case in the other bugfix, this job still
exists elsewhere in the system.

Now we verify that a job exists at all, and also that it is still
supposed to be run for the change in question when deciding whether
to keep it in the build-set for the change.

Change-Id: I0829d42612e7e74d789c60d0570e5bdf9ec47937
diff --git a/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml b/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml
new file mode 100644
index 0000000..ad3f666
--- /dev/null
+++ b/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml
@@ -0,0 +1,62 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: gate
+    manager: DependentPipelineManager
+    failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+jobs:
+  - name: ^.*-merge$
+    failure-message: Unable to merge change
+    hold-following-changes: true
+  - name: project1-project2-integration
+    queue-name: integration
+
+projects:
+  - name: org/project1
+    check:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+    gate:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+
+  - name: org/project2
+    check:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration
+    gate:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index f83eda4..61a2d09 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2440,6 +2440,60 @@
         # Ensure the removed job was not included in the report.
         self.assertNotIn('project-test1', A.messages[0])
 
+    def test_live_reconfiguration_shared_queue(self):
+        # Test that a change with a failing job which was removed from
+        # this project but otherwise still exists in the system does
+        # not disrupt reconfiguration.
+
+        self.worker.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+
+        self.worker.addFailTest('project1-project2-integration', A)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.worker.release('.*-merge')
+        self.waitUntilSettled()
+        self.worker.release('project1-project2-integration')
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 0)
+
+        self.assertEqual(self.getJobFromHistory('project1-merge').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory(
+            'project1-project2-integration').result, 'FAILURE')
+        self.assertEqual(len(self.history), 2)
+
+        # Remove the integration job.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-live-'
+                        'reconfiguration-shared-queue.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(self.getJobFromHistory('project1-merge').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project1-test1').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project1-test2').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory(
+            'project1-project2-integration').result, 'FAILURE')
+        self.assertEqual(len(self.history), 4)
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 1)
+        self.assertIn('Build succeeded', A.messages[0])
+        # Ensure the removed job was not included in the report.
+        self.assertNotIn('project1-project2-integration', A.messages[0])
+
     def test_live_reconfiguration_functions(self):
         "Test live reconfiguration with a custom function"
         self.worker.registerFunction('build:node-project-test1:debian')