diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 257b95d..f0235a6 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -103,7 +103,14 @@
             repo.fetchFrom(zuul_remote, ref)
             self.log.debug("Fetched ref %s from %s", ref, project)
             return True
-        except (ValueError, GitCommandError):
+        except ValueError:
+            self.log.debug("Project %s in Zuul does not have ref %s",
+                           project, ref)
+            return False
+        except GitCommandError as error:
+            # Bail out if fetch fails due to infrastructure reasons
+            if error.stderr.startswith('fatal: unable to access'):
+                raise
             self.log.debug("Project %s in Zuul does not have ref %s",
                            project, ref)
             return False