Move semaphore tests to their own class

Create a dedicated config directory for the semaphore tests and
remove them from the single-tenant configuration.

Create a simplified form of commitLayoutUpdate which accepts a
path to a replacement zuul.yaml and commits it to the specified
config repository to aid in reconfiguration tests.  The existing
similar methods rely on an entire shadow git repository which
requires additional git filesystem operations in tests.

Change-Id: I0f8e99b6ad262ece5a5649a480e0393872761903
diff --git a/tests/base.py b/tests/base.py
index 9c53846..c161b15 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2054,7 +2054,31 @@
             commit_data)
         return before
 
+    def commitConfigUpdate(self, project_name, source_name):
+        """Commit an update to zuul.yaml
+
+        This overwrites the zuul.yaml in the specificed project with
+        the contents specified.
+
+        :arg str project_name: The name of the project containing
+            zuul.yaml (e.g., common-config)
+
+        :arg str source_name: The path to the file (underneath the
+            test fixture directory) whose contents should be used to
+            replace zuul.yaml.
+        """
+
+        source_path = os.path.join(FIXTURE_DIR, source_name)
+        commit_data = {}
+        with open(source_path, 'r') as nt:
+            commit_data['zuul.yaml'] = nt.read()
+        before = self.addCommitToRepo(
+            project_name, 'Pulling content from %s' % source_name,
+            commit_data)
+        return before
+
     def addEvent(self, connection, event):
+
         """Inject a Fake (Gerrit) event.
 
         This method accepts a JSON-encoded event and simulates Zuul
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/project-test1.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/project-test1.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/project-test1.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/project-test1.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test1.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test1.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test1.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test1.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test2.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test2.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test2.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test2.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test1.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test1.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test1.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test1.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test2.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test2.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test2.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test2.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/zuul.yaml b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
similarity index 85%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/zuul.yaml
rename to tests/fixtures/config/semaphore/git/common-config/zuul.yaml
index 6a8906b..9d1cacf 100644
--- a/tests/fixtures/config/single-tenant/git/layout-semaphore/zuul.yaml
+++ b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
@@ -11,6 +11,14 @@
       gerrit:
         verified: -1
 
+# TODOv3(jeblair, tobiash): make semaphore definitions required, which
+# will cause these tests to fail until we define test-semaphore
+# here.
+
+- semaphore:
+    name: test-semaphore-two
+    max: 2
+
 - job:
     name: project-test1
 
@@ -45,7 +53,3 @@
         - project-test1
         - semaphore-two-test1
         - semaphore-two-test2
-
-- semaphore:
-    name: test-semaphore-two
-    max: 2
diff --git a/tests/fixtures/config/semaphore/git/org_project/README b/tests/fixtures/config/semaphore/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore/git/org_project1/README b/tests/fixtures/config/semaphore/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore/main.yaml b/tests/fixtures/config/semaphore/main.yaml
new file mode 100644
index 0000000..5f57245
--- /dev/null
+++ b/tests/fixtures/config/semaphore/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+          - org/project1
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/zuul.yaml b/tests/fixtures/config/semaphore/zuul-reconfiguration.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/zuul.yaml
rename to tests/fixtures/config/semaphore/zuul-reconfiguration.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/playbooks/project-test1.yaml
deleted file mode 100644
index f679dce..0000000
--- a/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/playbooks/project-test1.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-- hosts: all
-  tasks: []
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 8723766..60fd0ca 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2262,246 +2262,6 @@
         self.assertEqual('https://server/job/project-test2/0/',
                          status_jobs[2]['report_url'])
 
-    def test_semaphore_one(self):
-        "Test semaphores with max=1 (mutex)"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
-        self.assertEqual(self.builds[2].name, 'project-test1')
-
-        self.executor_server.release('semaphore-one-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.executor_server.release('semaphore-one-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.executor_server.release('semaphore-one-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.executor_server.release('semaphore-one-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 2)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-
-        self.assertEqual(A.reported, 1)
-        self.assertEqual(B.reported, 1)
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-    def test_semaphore_two(self):
-        "Test semaphores with max>1"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
-        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
-        self.assertFalse('test-semaphore-two' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 4)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
-        self.assertEqual(self.builds[3].name, 'project-test1')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 2)
-
-        self.executor_server.release('semaphore-two-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 4)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
-        self.assertEqual(self.builds[2].name, 'project-test1')
-        self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 2)
-
-        self.executor_server.release('semaphore-two-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 4)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
-        self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 2)
-
-        self.executor_server.release('semaphore-two-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 1)
-
-        self.executor_server.release('semaphore-two-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 2)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertFalse('test-semaphore-two' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-
-        self.assertEqual(A.reported, 1)
-        self.assertEqual(B.reported, 1)
-
-    def test_semaphore_abandon(self):
-        "Test abandon with job semaphores"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-
-        tenant = self.sched.abide.tenants.get('openstack')
-        check_pipeline = tenant.layout.pipelines['check']
-
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
-        self.waitUntilSettled()
-
-        # The check pipeline should be empty
-        items = check_pipeline.getAllItems()
-        self.assertEqual(len(items), 0)
-
-        # The semaphore should be released
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-        self.waitUntilSettled()
-
-    def test_semaphore_reconfigure(self):
-        "Test reconfigure with job semaphores"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        # reconfigure without layout change
-        self.sched.reconfigure(self.config)
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        # semaphore still must be held
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.updateConfigLayout('layout-semaphore-reconfiguration')
-        self.sched.reconfigure(self.config)
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.release('project-test1')
-        self.waitUntilSettled()
-
-        # There should be no builds anymore
-        self.assertEqual(len(self.builds), 0)
-
-        # The semaphore should be released
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
     def test_live_reconfiguration(self):
         "Test that live reconfiguration works"
         self.executor_server.hold_jobs_in_build = True
@@ -5089,6 +4849,231 @@
         self.waitUntilSettled()
 
 
+class TestSemaphore(ZuulTestCase):
+    tenant_config_file = 'config/semaphore/main.yaml'
+
+    def test_semaphore_one(self):
+        "Test semaphores with max=1 (mutex)"
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
+        self.assertEqual(self.builds[2].name, 'project-test1')
+
+        self.executor_server.release('semaphore-one-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.executor_server.release('semaphore-one-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.executor_server.release('semaphore-one-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.executor_server.release('semaphore-one-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+
+        self.waitUntilSettled()
+        self.assertEqual(len(self.builds), 0)
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(B.reported, 1)
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+    def test_semaphore_two(self):
+        "Test semaphores with max>1"
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        self.assertFalse('test-semaphore-two' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 4)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
+        self.assertEqual(self.builds[3].name, 'project-test1')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 2)
+
+        self.executor_server.release('semaphore-two-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 4)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
+        self.assertEqual(self.builds[2].name, 'project-test1')
+        self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 2)
+
+        self.executor_server.release('semaphore-two-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 4)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
+        self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 2)
+
+        self.executor_server.release('semaphore-two-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 1)
+
+        self.executor_server.release('semaphore-two-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertFalse('test-semaphore-two' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+
+        self.waitUntilSettled()
+        self.assertEqual(len(self.builds), 0)
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(B.reported, 1)
+
+    def test_semaphore_abandon(self):
+        "Test abandon with job semaphores"
+        self.executor_server.hold_jobs_in_build = True
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        check_pipeline = tenant.layout.pipelines['check']
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
+        self.waitUntilSettled()
+
+        # The check pipeline should be empty
+        items = check_pipeline.getAllItems()
+        self.assertEqual(len(items), 0)
+
+        # The semaphore should be released
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+    def test_semaphore_reconfigure(self):
+        "Test reconfigure with job semaphores"
+        self.executor_server.hold_jobs_in_build = True
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        # reconfigure without layout change
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        # semaphore still must be held
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.commitConfigUpdate(
+            'common-config',
+            'config/semaphore/zuul-reconfiguration.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.release('project-test1')
+        self.waitUntilSettled()
+
+        # There should be no builds anymore
+        self.assertEqual(len(self.builds), 0)
+
+        # The semaphore should be released
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+
 class TestSemaphoreMultiTenant(ZuulTestCase):
     tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'