diff --git a/tests/base.py b/tests/base.py
index d6c36a8..cb1cf11 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2055,7 +2055,31 @@
             commit_data)
         return before
 
+    def commitConfigUpdate(self, project_name, source_name):
+        """Commit an update to zuul.yaml
+
+        This overwrites the zuul.yaml in the specificed project with
+        the contents specified.
+
+        :arg str project_name: The name of the project containing
+            zuul.yaml (e.g., common-config)
+
+        :arg str source_name: The path to the file (underneath the
+            test fixture directory) whose contents should be used to
+            replace zuul.yaml.
+        """
+
+        source_path = os.path.join(FIXTURE_DIR, source_name)
+        commit_data = {}
+        with open(source_path, 'r') as nt:
+            commit_data['zuul.yaml'] = nt.read()
+        before = self.addCommitToRepo(
+            project_name, 'Pulling content from %s' % source_name,
+            commit_data)
+        return before
+
     def addEvent(self, connection, event):
+
         """Inject a Fake (Gerrit) event.
 
         This method accepts a JSON-encoded event and simulates Zuul
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/project-test1.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/project-test1.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/project-test1.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/project-test1.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test1.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test1.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test1.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test1.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test2.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test2.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-one-test2.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-one-test2.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test1.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test1.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test1.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test1.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test2.yaml b/tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test2.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/playbooks/semaphore-two-test2.yaml
rename to tests/fixtures/config/semaphore/git/common-config/playbooks/semaphore-two-test2.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore/zuul.yaml b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
similarity index 85%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore/zuul.yaml
rename to tests/fixtures/config/semaphore/git/common-config/zuul.yaml
index 6a8906b..9d1cacf 100644
--- a/tests/fixtures/config/single-tenant/git/layout-semaphore/zuul.yaml
+++ b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
@@ -11,6 +11,14 @@
       gerrit:
         verified: -1
 
+# TODOv3(jeblair, tobiash): make semaphore definitions required, which
+# will cause these tests to fail until we define test-semaphore
+# here.
+
+- semaphore:
+    name: test-semaphore-two
+    max: 2
+
 - job:
     name: project-test1
 
@@ -45,7 +53,3 @@
         - project-test1
         - semaphore-two-test1
         - semaphore-two-test2
-
-- semaphore:
-    name: test-semaphore-two
-    max: 2
diff --git a/tests/fixtures/config/semaphore/git/org_project/README b/tests/fixtures/config/semaphore/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore/git/org_project1/README b/tests/fixtures/config/semaphore/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore/main.yaml b/tests/fixtures/config/semaphore/main.yaml
new file mode 100644
index 0000000..5f57245
--- /dev/null
+++ b/tests/fixtures/config/semaphore/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+          - org/project1
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/zuul.yaml b/tests/fixtures/config/semaphore/zuul-reconfiguration.yaml
similarity index 100%
rename from tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/zuul.yaml
rename to tests/fixtures/config/semaphore/zuul-reconfiguration.yaml
diff --git a/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/playbooks/project-test1.yaml
deleted file mode 100644
index f679dce..0000000
--- a/tests/fixtures/config/single-tenant/git/layout-semaphore-reconfiguration/playbooks/project-test1.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-- hosts: all
-  tasks: []
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 8723766..60fd0ca 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2262,246 +2262,6 @@
         self.assertEqual('https://server/job/project-test2/0/',
                          status_jobs[2]['report_url'])
 
-    def test_semaphore_one(self):
-        "Test semaphores with max=1 (mutex)"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
-        self.assertEqual(self.builds[2].name, 'project-test1')
-
-        self.executor_server.release('semaphore-one-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.executor_server.release('semaphore-one-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.executor_server.release('semaphore-one-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.executor_server.release('semaphore-one-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 2)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-
-        self.assertEqual(A.reported, 1)
-        self.assertEqual(B.reported, 1)
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-    def test_semaphore_two(self):
-        "Test semaphores with max>1"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
-        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
-        self.assertFalse('test-semaphore-two' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 4)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
-        self.assertEqual(self.builds[3].name, 'project-test1')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 2)
-
-        self.executor_server.release('semaphore-two-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 4)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
-        self.assertEqual(self.builds[2].name, 'project-test1')
-        self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 2)
-
-        self.executor_server.release('semaphore-two-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 4)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
-        self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 2)
-
-        self.executor_server.release('semaphore-two-test1')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 3)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
-        self.assertTrue('test-semaphore-two' in
-                        tenant.semaphore_handler.semaphores)
-        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
-            'test-semaphore-two', [])), 1)
-
-        self.executor_server.release('semaphore-two-test2')
-        self.waitUntilSettled()
-
-        self.assertEqual(len(self.builds), 2)
-        self.assertEqual(self.builds[0].name, 'project-test1')
-        self.assertEqual(self.builds[1].name, 'project-test1')
-        self.assertFalse('test-semaphore-two' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-
-        self.assertEqual(A.reported, 1)
-        self.assertEqual(B.reported, 1)
-
-    def test_semaphore_abandon(self):
-        "Test abandon with job semaphores"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-
-        tenant = self.sched.abide.tenants.get('openstack')
-        check_pipeline = tenant.layout.pipelines['check']
-
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
-        self.waitUntilSettled()
-
-        # The check pipeline should be empty
-        items = check_pipeline.getAllItems()
-        self.assertEqual(len(items), 0)
-
-        # The semaphore should be released
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.executor_server.hold_jobs_in_build = False
-        self.executor_server.release()
-        self.waitUntilSettled()
-
-    def test_semaphore_reconfigure(self):
-        "Test reconfigure with job semaphores"
-        self.updateConfigLayout('layout-semaphore')
-        self.sched.reconfigure(self.config)
-
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.hold_jobs_in_build = True
-
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.waitUntilSettled()
-
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        # reconfigure without layout change
-        self.sched.reconfigure(self.config)
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        # semaphore still must be held
-        self.assertTrue('test-semaphore' in
-                        tenant.semaphore_handler.semaphores)
-
-        self.updateConfigLayout('layout-semaphore-reconfiguration')
-        self.sched.reconfigure(self.config)
-        self.waitUntilSettled()
-        tenant = self.sched.abide.tenants.get('openstack')
-
-        self.executor_server.release('project-test1')
-        self.waitUntilSettled()
-
-        # There should be no builds anymore
-        self.assertEqual(len(self.builds), 0)
-
-        # The semaphore should be released
-        self.assertFalse('test-semaphore' in
-                         tenant.semaphore_handler.semaphores)
-
     def test_live_reconfiguration(self):
         "Test that live reconfiguration works"
         self.executor_server.hold_jobs_in_build = True
@@ -5089,6 +4849,231 @@
         self.waitUntilSettled()
 
 
+class TestSemaphore(ZuulTestCase):
+    tenant_config_file = 'config/semaphore/main.yaml'
+
+    def test_semaphore_one(self):
+        "Test semaphores with max=1 (mutex)"
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
+        self.assertEqual(self.builds[2].name, 'project-test1')
+
+        self.executor_server.release('semaphore-one-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.executor_server.release('semaphore-one-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.executor_server.release('semaphore-one-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.executor_server.release('semaphore-one-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+
+        self.waitUntilSettled()
+        self.assertEqual(len(self.builds), 0)
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(B.reported, 1)
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+    def test_semaphore_two(self):
+        "Test semaphores with max>1"
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        self.assertFalse('test-semaphore-two' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 4)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
+        self.assertEqual(self.builds[3].name, 'project-test1')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 2)
+
+        self.executor_server.release('semaphore-two-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 4)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
+        self.assertEqual(self.builds[2].name, 'project-test1')
+        self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 2)
+
+        self.executor_server.release('semaphore-two-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 4)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
+        self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 2)
+
+        self.executor_server.release('semaphore-two-test1')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 3)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
+        self.assertTrue('test-semaphore-two' in
+                        tenant.semaphore_handler.semaphores)
+        self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
+            'test-semaphore-two', [])), 1)
+
+        self.executor_server.release('semaphore-two-test2')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+        self.assertEqual(self.builds[0].name, 'project-test1')
+        self.assertEqual(self.builds[1].name, 'project-test1')
+        self.assertFalse('test-semaphore-two' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+
+        self.waitUntilSettled()
+        self.assertEqual(len(self.builds), 0)
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(B.reported, 1)
+
+    def test_semaphore_abandon(self):
+        "Test abandon with job semaphores"
+        self.executor_server.hold_jobs_in_build = True
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        check_pipeline = tenant.layout.pipelines['check']
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
+        self.waitUntilSettled()
+
+        # The check pipeline should be empty
+        items = check_pipeline.getAllItems()
+        self.assertEqual(len(items), 0)
+
+        # The semaphore should be released
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+    def test_semaphore_reconfigure(self):
+        "Test reconfigure with job semaphores"
+        self.executor_server.hold_jobs_in_build = True
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        # reconfigure without layout change
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        # semaphore still must be held
+        self.assertTrue('test-semaphore' in
+                        tenant.semaphore_handler.semaphores)
+
+        self.commitConfigUpdate(
+            'common-config',
+            'config/semaphore/zuul-reconfiguration.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+        tenant = self.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.release('project-test1')
+        self.waitUntilSettled()
+
+        # There should be no builds anymore
+        self.assertEqual(len(self.builds), 0)
+
+        # The semaphore should be released
+        self.assertFalse('test-semaphore' in
+                         tenant.semaphore_handler.semaphores)
+
+
 class TestSemaphoreMultiTenant(ZuulTestCase):
     tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'