diff --git a/requirements.txt b/requirements.txt
index 7057c5a..47c0f5e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,6 @@
 git+https://github.com/sigmavirus24/github3.py.git@develop#egg=Github3.py
 PyYAML>=3.1.0
 Paste
-WebOb>=1.2.3
 paramiko>=2.0.1
 GitPython>=2.1.8
 python-daemon>=2.0.4,<2.1.0
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 7f9e651..e150a47 100644
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -22,9 +22,6 @@
 import urllib
 import time
 import socket
-from unittest import skip
-
-import webob
 
 import zuul.web
 
@@ -212,21 +209,6 @@
         f = urllib.request.urlopen(req)
         self.assertEqual(f.read(), public_pem)
 
-    @skip("This may not apply to zuul-web")
-    def test_web_custom_handler(self):
-        def custom_handler(path, tenant_name, request):
-            return webob.Response(body='ok')
-
-        self.webapp.register_path('/custom', custom_handler)
-        req = urllib.request.Request(
-            "http://localhost:%s/custom" % self.port)
-        f = urllib.request.urlopen(req)
-        self.assertEqual(b'ok', f.read())
-
-        self.webapp.unregister_path('/custom')
-        self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, req)
-
-    @skip("This returns a 500")
     def test_web_404_on_unknown_tenant(self):
         req = urllib.request.Request(
             "http://localhost:{}/non-tenant/status".format(self.port))
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 270b91c..df6336d 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1754,20 +1754,22 @@
                             config_path)
         return config_path
 
-    def loadConfig(self, config_path, project_key_dir):
-        abide = model.Abide()
-
+    def readConfig(self, config_path):
         config_path = self.expandConfigPath(config_path)
         with open(config_path) as config_file:
             self.log.info("Loading configuration from %s" % (config_path,))
             data = yaml.safe_load(config_file)
-        config = model.UnparsedAbideConfig()
-        config.extend(data)
         base = os.path.dirname(os.path.realpath(config_path))
+        unparsed_abide = model.UnparsedAbideConfig(base)
+        unparsed_abide.extend(data)
+        return unparsed_abide
 
-        for conf_tenant in config.tenants:
+    def loadConfig(self, unparsed_abide, project_key_dir):
+        abide = model.Abide()
+        for conf_tenant in unparsed_abide.tenants:
             # When performing a full reload, do not use cached data.
-            tenant = self.tenant_parser.fromYaml(base, project_key_dir,
+            tenant = self.tenant_parser.fromYaml(unparsed_abide.base,
+                                                 project_key_dir,
                                                  conf_tenant, old_tenant=None)
             abide.tenants[tenant.name] = tenant
         return abide
diff --git a/zuul/model.py b/zuul/model.py
index 763eb66..bd4f4d1 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -2446,8 +2446,10 @@
     An Abide is a collection of tenants.
     """
 
-    def __init__(self):
+    def __init__(self, base=None):
         self.tenants = []
+        self.known_tenants = set()
+        self.base = base
 
     def extend(self, conf):
         if isinstance(conf, UnparsedAbideConfig):
@@ -2465,6 +2467,8 @@
             key, value = list(item.items())[0]
             if key == 'tenant':
                 self.tenants.append(value)
+                if 'name' in value:
+                    self.known_tenants.add(value['name'])
             else:
                 raise ConfigItemUnknownError()
 
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 083cb12..606cd04 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -246,6 +246,7 @@
         self.result_event_queue = queue.Queue()
         self.management_event_queue = zuul.lib.queue.MergedQueue()
         self.abide = model.Abide()
+        self.unparsed_abide = model.UnparsedAbideConfig()
 
         if not testonly:
             time_dir = self._get_time_database_dir()
@@ -550,8 +551,10 @@
             self.log.info("Full reconfiguration beginning")
             loader = configloader.ConfigLoader(
                 self.connections, self, self.merger)
+            self.unparsed_abide = loader.readConfig(
+                self.config.get('scheduler', 'tenant_config'))
             abide = loader.loadConfig(
-                self.config.get('scheduler', 'tenant_config'),
+                self.unparsed_abide,
                 self._get_project_key_dir())
             for tenant in abide.tenants.values():
                 self._reconfigureTenant(tenant)
@@ -1149,6 +1152,8 @@
         data['pipelines'] = pipelines
         tenant = self.abide.tenants.get(tenant_name)
         if not tenant:
+            if tenant_name not in self.unparsed_abide.known_tenants:
+                return json.dumps({"message": "Unknown tenant"})
             self.log.warning("Tenant %s isn't loaded" % tenant_name)
             return json.dumps(
                 {"message": "Tenant %s isn't ready" % tenant_name})
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index 08df42a..41b1b81 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -171,6 +171,8 @@
             self.cache[tenant] = json.loads(job.data[0])
             self.cache_time[tenant] = time.time()
         payload = self.cache[tenant]
+        if payload.get('message') == 'Unknown tenant':
+            return web.HTTPNotFound()
         if result_filter:
             payload = result_filter.filterPayload(payload)
         resp = web.json_response(payload)