Return 404 on unknown tenants

These aren't exceptional, so don't log them as such.

Change-Id: I5eb9b0ff2732fb55d21ea066f1b896507b5dba1f
diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py
index 5b6950b..c06fc93 100644
--- a/tests/unit/test_webapp.py
+++ b/tests/unit/test_webapp.py
@@ -110,3 +110,10 @@
 
         self.webapp.unregister_path('/custom')
         self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, req)
+
+    def test_webapp_404_on_unknown_tenant(self):
+        req = urllib.request.Request(
+            "http://localhost:{}/non-tenant/status.json".format(self.port))
+        e = self.assertRaises(
+            urllib.error.HTTPError, urllib.request.urlopen, req)
+        self.assertEqual(404, e.code)
diff --git a/zuul/webapp.py b/zuul/webapp.py
index b9129b8..134fb3c 100644
--- a/zuul/webapp.py
+++ b/zuul/webapp.py
@@ -153,6 +153,8 @@
             return webob.Response(body=self.cache[tenant_name],
                                   content_type='application/json',
                                   charset='utf8')
+        if tenant_name not in self.scheduler.abide.tenants:
+            raise webob.exc.HTTPNotFound()
         return self._response_with_status_cache(func, tenant_name)
 
     def change(self, path, tenant_name, request):