Merge "Add check for ref being a string before applying regex"
diff --git a/doc/source/cloner.rst b/doc/source/cloner.rst
index bb33f82..2ddf0b5 100644
--- a/doc/source/cloner.rst
+++ b/doc/source/cloner.rst
@@ -75,3 +75,15 @@
 projects::
 
   zuul-cloner project project/plugins/plugin1
+
+Cached repositories
+-------------------
+
+The ``--cache-dir`` option can be used to reduce network traffic by
+cloning from a local repository which may not be up to date.
+
+If the ``--cache-dir`` option is supplied, zuul-cloner will start by
+cloning any projects it processes from those found in that directory.
+The URL of origin remote of the resulting clone will be reset to use
+the ``git_base_url`` and then the remote will be updated so that the
+repository has all the information in the upstream repository.
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index cdfa4df..683ca00 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -978,6 +978,10 @@
      check:
       - foobar-extra-special-job
 
+Individual jobs may optionally be added to pipelines (e.g. check,
+gate, et cetera) for a project, in addtion to those provided by
+templates.
+
 The order of the jobs listed in the project (which only affects the
 order of jobs listed on the report) will be the jobs from each
 template in the order listed, followed by any jobs individually listed
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index 01f8bd3..5d155af 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -50,7 +50,7 @@
         }, options);
 
         var collapsed_exceptions = [];
-        var current_filter = read_cookie('zuul_filter_string', current_filter);
+        var current_filter = read_cookie('zuul_filter_string', '');
         var $jq;
 
         var xhr,
@@ -111,6 +111,7 @@
                     );
                 }
 
+                $job_line.append($('<div style="clear: both"></div>'));
                 return $job_line;
             },
 
@@ -266,9 +267,21 @@
 
                 var $change_link = $('<small />');
                 if (change.url !== null) {
-                    $change_link.append(
-                        $('<a />').attr('href', change.url).text(change.id)
-                    );
+                    if (/^[0-9a-f]{40}$/.test(change.id)) {
+                        var change_id_short = change.id.slice(0, 7);
+                        $change_link.append(
+                            $('<a />').attr('href', change.url).append(
+                                $('<abbr />')
+                                    .attr('title', change.id)
+                                    .text(change_id_short)
+                            )
+                        );
+                    }
+                    else {
+                        $change_link.append(
+                            $('<a />').attr('href', change.url).text(change.id)
+                        );
+                    }
                 }
                 else {
                     $change_link.text(change_id);
diff --git a/requirements.txt b/requirements.txt
index eabcef3..50726c0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,8 +14,6 @@
 voluptuous>=0.7
 gear>=0.5.4,<1.0.0
 apscheduler>=2.1.1,<3.0
-python-swiftclient>=1.6
-python-keystoneclient>=0.4.2
 PrettyTable>=0.6,<0.8
 babel>=1.0
 six>=1.6.0
diff --git a/test-requirements.txt b/test-requirements.txt
index 99ada89..5192de7 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -6,7 +6,9 @@
 docutils==0.9.1
 discover
 fixtures>=0.3.14
+python-keystoneclient>=0.4.2
 python-subunit
+python-swiftclient>=1.6
 testrepository>=0.0.17
 testtools>=0.9.32
 sphinxcontrib-programoutput
diff --git a/tests/base.py b/tests/base.py
index 753bc5e..46c7087 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -56,6 +56,7 @@
 
 FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
                            'fixtures')
+USE_TEMPDIR = True
 
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s %(name)-32s '
@@ -165,7 +166,7 @@
         if files:
             fn = files[0]
         else:
-            fn = '%s-%s' % (self.branch, self.number)
+            fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
         msg = self.subject + '-' + str(self.latest_patchset)
         c = self.add_fake_change_to_repo(msg, fn, large)
         ps_files = [{'file': '/COMMIT_MSG',
@@ -821,8 +822,11 @@
                 level=logging.DEBUG,
                 format='%(asctime)s %(name)-32s '
                 '%(levelname)-8s %(message)s'))
-        tmp_root = self.useFixture(fixtures.TempDir(
-            rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+        if USE_TEMPDIR:
+            tmp_root = self.useFixture(fixtures.TempDir(
+                    rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+        else:
+            tmp_root = os.environ.get("ZUUL_TEST_ROOT")
         self.test_root = os.path.join(tmp_root, "zuul-test")
         self.upstream_root = os.path.join(self.test_root, "upstream")
         self.git_root = os.path.join(self.test_root, "git")
@@ -844,6 +848,9 @@
         self.init_repo("org/project1")
         self.init_repo("org/project2")
         self.init_repo("org/project3")
+        self.init_repo("org/project4")
+        self.init_repo("org/project5")
+        self.init_repo("org/project6")
         self.init_repo("org/one-job-project")
         self.init_repo("org/nonvoting-project")
         self.init_repo("org/templated-project")
@@ -998,15 +1005,26 @@
         master = repo.create_head('master')
         repo.create_tag('init')
 
-        mp = repo.create_head('mp')
-        repo.head.reference = mp
+        repo.head.reference = master
+        repo.head.reset(index=True, working_tree=True)
+        repo.git.clean('-x', '-f', '-d')
+
+        self.create_branch(project, 'mp')
+
+    def create_branch(self, project, branch):
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo.init(path)
+        fn = os.path.join(path, 'README')
+
+        branch_head = repo.create_head(branch)
+        repo.head.reference = branch_head
         f = open(fn, 'a')
-        f.write("test mp\n")
+        f.write("test %s\n" % branch)
         f.close()
         repo.index.add([fn])
-        repo.index.commit('mp commit')
+        repo.index.commit('%s commit' % branch)
 
-        repo.head.reference = master
+        repo.head.reference = repo.heads['master']
         repo.head.reset(index=True, working_tree=True)
         repo.git.clean('-x', '-f', '-d')
 
diff --git a/tests/fixtures/layout-gating.yaml b/tests/fixtures/layout-cloner.yaml
similarity index 65%
rename from tests/fixtures/layout-gating.yaml
rename to tests/fixtures/layout-cloner.yaml
index a544a80..e840ed9 100644
--- a/tests/fixtures/layout-gating.yaml
+++ b/tests/fixtures/layout-cloner.yaml
@@ -22,8 +22,24 @@
 
   - name: org/project1
     gate:
-        - project1-project2-integration
+        - integration
 
   - name: org/project2
     gate:
-        - project1-project2-integration
+        - integration
+
+  - name: org/project3
+    gate:
+        - integration
+
+  - name: org/project4
+    gate:
+        - integration
+
+  - name: org/project5
+    gate:
+        - integration
+
+  - name: org/project6
+    gate:
+        - integration
diff --git a/tests/fixtures/layouts/bad_merge_failure.yaml b/tests/fixtures/layouts/bad_merge_failure.yaml
index 313d23b..fc6854e 100644
--- a/tests/fixtures/layouts/bad_merge_failure.yaml
+++ b/tests/fixtures/layouts/bad_merge_failure.yaml
@@ -10,6 +10,7 @@
     failure:
       gerrit:
         verified: -1
+    # merge-failure-message needs a string.
     merge-failure-message:
 
   - name: gate
diff --git a/tests/fixtures/layouts/bad_pipelines b/tests/fixtures/layouts/bad_pipelines
deleted file mode 100644
index f627208..0000000
--- a/tests/fixtures/layouts/bad_pipelines
+++ /dev/null
@@ -1 +0,0 @@
-pipelines:
diff --git a/tests/fixtures/layouts/bad_pipelines1.yaml b/tests/fixtures/layouts/bad_pipelines1.yaml
index da90933..09638bc 100644
--- a/tests/fixtures/layouts/bad_pipelines1.yaml
+++ b/tests/fixtures/layouts/bad_pipelines1.yaml
@@ -1,2 +1,2 @@
+# Pipelines completely missing. At least one is required.
 pipelines:
-
diff --git a/tests/fixtures/layouts/bad_pipelines10.yaml b/tests/fixtures/layouts/bad_pipelines10.yaml
index 5248c17..ddde946 100644
--- a/tests/fixtures/layouts/bad_pipelines10.yaml
+++ b/tests/fixtures/layouts/bad_pipelines10.yaml
@@ -4,4 +4,5 @@
 
 projects:
   - name: foo
-    merge-mode: foo
\ No newline at end of file
+    # merge-mode must be one of merge, merge-resolve, cherry-pick.
+    merge-mode: foo
diff --git a/tests/fixtures/layouts/bad_pipelines2.yaml b/tests/fixtures/layouts/bad_pipelines2.yaml
index e75a561..fc1e154 100644
--- a/tests/fixtures/layouts/bad_pipelines2.yaml
+++ b/tests/fixtures/layouts/bad_pipelines2.yaml
@@ -1,4 +1,5 @@
 pipelines:
+  # name is required for pipelines
   - noname: check
     manager: IndependentPipelineManager
 
diff --git a/tests/fixtures/layouts/bad_pipelines3.yaml b/tests/fixtures/layouts/bad_pipelines3.yaml
index 0c11a85..93ac266 100644
--- a/tests/fixtures/layouts/bad_pipelines3.yaml
+++ b/tests/fixtures/layouts/bad_pipelines3.yaml
@@ -1,5 +1,7 @@
 pipelines:
   - name: check
+    # The manager must be one of IndependentPipelineManager
+    # or DependentPipelineManager
     manager: NonexistentPipelineManager
 
 projects:
diff --git a/tests/fixtures/layouts/bad_pipelines4.yaml b/tests/fixtures/layouts/bad_pipelines4.yaml
index 7f58024..3a91604 100644
--- a/tests/fixtures/layouts/bad_pipelines4.yaml
+++ b/tests/fixtures/layouts/bad_pipelines4.yaml
@@ -3,6 +3,7 @@
     manager: IndependentPipelineManager
     trigger:
       gerrit:
+        # non-event is not a valid gerrit event
         - event: non-event
 
 projects:
diff --git a/tests/fixtures/layouts/bad_pipelines5.yaml b/tests/fixtures/layouts/bad_pipelines5.yaml
index 929c1a9..f95a78e 100644
--- a/tests/fixtures/layouts/bad_pipelines5.yaml
+++ b/tests/fixtures/layouts/bad_pipelines5.yaml
@@ -3,6 +3,7 @@
     manager: IndependentPipelineManager
     trigger:
       gerrit:
+        # event is a required item but it is missing.
         - approval:
             - approved: 1
 
diff --git a/tests/fixtures/layouts/bad_pipelines6.yaml b/tests/fixtures/layouts/bad_pipelines6.yaml
index 6dcdaf3..aa91c77 100644
--- a/tests/fixtures/layouts/bad_pipelines6.yaml
+++ b/tests/fixtures/layouts/bad_pipelines6.yaml
@@ -4,6 +4,7 @@
     trigger:
       gerrit:
         - event: comment-added
+          # approved is not a valid entry. Should be approval.
           approved: 1
 
 projects:
diff --git a/tests/fixtures/layouts/bad_pipelines7.yaml b/tests/fixtures/layouts/bad_pipelines7.yaml
index 7517b9a..e2db495 100644
--- a/tests/fixtures/layouts/bad_pipelines7.yaml
+++ b/tests/fixtures/layouts/bad_pipelines7.yaml
@@ -1,4 +1,5 @@
 pipelines:
+  # The pipeline must have a name.
   - manager: IndependentPipelineManager
 
 projects:
diff --git a/tests/fixtures/layouts/bad_pipelines8.yaml b/tests/fixtures/layouts/bad_pipelines8.yaml
index eeab038..9c5918e 100644
--- a/tests/fixtures/layouts/bad_pipelines8.yaml
+++ b/tests/fixtures/layouts/bad_pipelines8.yaml
@@ -1,4 +1,5 @@
 pipelines:
+  # The pipeline must have a manager
   - name: check
 
 projects:
diff --git a/tests/fixtures/layouts/bad_pipelines9.yaml b/tests/fixtures/layouts/bad_pipelines9.yaml
index ebb2e1f..89307d5 100644
--- a/tests/fixtures/layouts/bad_pipelines9.yaml
+++ b/tests/fixtures/layouts/bad_pipelines9.yaml
@@ -1,4 +1,5 @@
 pipelines:
+  # Names must be unique.
   - name: check
     manager: IndependentPipelineManager
   - name: check
diff --git a/tests/fixtures/layouts/bad_projects1.yaml b/tests/fixtures/layouts/bad_projects1.yaml
index c210c43..e3d381f 100644
--- a/tests/fixtures/layouts/bad_projects1.yaml
+++ b/tests/fixtures/layouts/bad_projects1.yaml
@@ -4,6 +4,7 @@
 
 projects:
   - name: foo
+  # gate pipeline is not defined.
     gate:
       - test
 
diff --git a/tests/fixtures/layouts/bad_projects2.yaml b/tests/fixtures/layouts/bad_projects2.yaml
index b91ed9d..9291cc9 100644
--- a/tests/fixtures/layouts/bad_projects2.yaml
+++ b/tests/fixtures/layouts/bad_projects2.yaml
@@ -5,5 +5,6 @@
 projects:
   - name: foo
     check:
+      # Indentation is one level too deep on the last line.
       - test
         - foo
diff --git a/tests/fixtures/layouts/bad_swift.yaml b/tests/fixtures/layouts/bad_swift.yaml
index d8a8c3f..e79dca6 100644
--- a/tests/fixtures/layouts/bad_swift.yaml
+++ b/tests/fixtures/layouts/bad_swift.yaml
@@ -16,11 +16,10 @@
     swift:
       - name: logs
   - name: ^.*-merge$
+    # swift requires a name
     swift:
         container: merge_assets
     failure-message: Unable to merge change
-  - name: test-test
-    swift:
 
 projects:
   - name: test-org/test
diff --git a/tests/fixtures/layouts/bad_template1.yaml b/tests/fixtures/layouts/bad_template1.yaml
index 15822d1..cab17a1 100644
--- a/tests/fixtures/layouts/bad_template1.yaml
+++ b/tests/fixtures/layouts/bad_template1.yaml
@@ -10,7 +10,7 @@
 project-templates:
   - name: template-generic
     check:
-     # Template uses the 'project' parameter' which must
+     # Template uses the 'project' parameter' which must be provided
      - '{project}-merge'
 
 projects:
diff --git a/tests/test_cloner.py b/tests/test_cloner.py
index bb9d91f..ab2683d 100644
--- a/tests/test_cloner.py
+++ b/tests/test_cloner.py
@@ -41,21 +41,88 @@
         self.workspace_root = os.path.join(self.test_root, 'workspace')
 
         self.config.set('zuul', 'layout_config',
-                        'tests/fixtures/layout-gating.yaml')
+                        'tests/fixtures/layout-cloner.yaml')
         self.sched.reconfigure(self.config)
         self.registerJobs()
 
-    def test_cloner(self):
+    def getWorkspaceRepos(self, projects):
+        repos = {}
+        for project in projects:
+            repos[project] = git.Repo(
+                os.path.join(self.workspace_root, project))
+        return repos
+
+    def getUpstreamRepos(self, projects):
+        repos = {}
+        for project in projects:
+            repos[project] = git.Repo(
+                os.path.join(self.upstream_root, project))
+        return repos
+
+    def test_cache_dir(self):
+        projects = ['org/project1', 'org/project2']
+        cache_root = os.path.join(self.test_root, "cache")
+        for project in projects:
+            upstream_repo_path = os.path.join(self.upstream_root, project)
+            cache_repo_path = os.path.join(cache_root, project)
+            git.Repo.clone_from(upstream_repo_path, cache_repo_path)
+
+        self.worker.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        A.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(1, len(self.builds), "One build is running")
+
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+        B.setMerged()
+
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                cache_dir=cache_root,
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
+
+        work = self.getWorkspaceRepos(projects)
+        upstream_repo_path = os.path.join(self.upstream_root, 'org/project1')
+        self.assertEquals(work['org/project1'].remotes.origin.url,
+                          upstream_repo_path,
+                          'workspace repo origin should be upstream, not cache')
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+    def test_one_branch(self):
         self.worker.hold_jobs_in_build = True
 
+        projects = ['org/project1', 'org/project2']
         A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
         B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
-
-        A.addPatchset(['project_one.txt'])
-        B.addPatchset(['project_two.txt'])
-        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
-
         A.addApproval('CRVW', 2)
         B.addApproval('CRVW', 2)
         self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
@@ -65,39 +132,352 @@
 
         self.assertEquals(2, len(self.builds), "Two builds are running")
 
-        a_zuul_ref = b_zuul_ref = None
-        for build in self.builds:
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
             self.log.debug("Build parameters: %s", build.parameters)
-            if build.parameters['ZUUL_CHANGE'] == '1':
-                a_zuul_ref = build.parameters['ZUUL_REF']
-                a_zuul_commit = build.parameters['ZUUL_COMMIT']
-            if build.parameters['ZUUL_CHANGE'] == '2':
-                b_zuul_ref = build.parameters['ZUUL_REF']
-                b_zuul_commit = build.parameters['ZUUL_COMMIT']
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
+
+            shutil.rmtree(self.workspace_root)
 
         self.worker.hold_jobs_in_build = False
         self.worker.release()
         self.waitUntilSettled()
 
-        # Repos setup, now test the cloner
-        for zuul_ref in [a_zuul_ref, b_zuul_ref]:
+    def test_multi_branch(self):
+        self.worker.hold_jobs_in_build = True
+        projects = ['org/project1', 'org/project2',
+                    'org/project3', 'org/project4']
+
+        self.create_branch('org/project2', 'stable/havana')
+        self.create_branch('org/project4', 'stable/havana')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+        A.addApproval('CRVW', 2)
+        B.addApproval('CRVW', 2)
+        C.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(3, len(self.builds), "Three builds are running")
+
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].
+                                 commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].
+                                 commit('stable/havana')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].
+                                 commit('master')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
             cloner = zuul.lib.cloner.Cloner(
                 git_base_url=self.upstream_root,
-                projects=['org/project1', 'org/project2'],
+                projects=projects,
                 workspace=self.workspace_root,
-                zuul_branch='master',
-                zuul_ref=zuul_ref,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
                 zuul_url=self.git_root,
-                branch='master',
-                clone_map_file=os.path.join(FIXTURE_DIR, 'clonemap.yaml')
-            )
+                )
             cloner.execute()
-            work_repo1 = git.Repo(os.path.join(self.workspace_root,
-                                               'org/project1'))
-            self.assertEquals(a_zuul_commit, str(work_repo1.commit('HEAD')))
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
 
-            work_repo2 = git.Repo(os.path.join(self.workspace_root,
-                                               'org/project2'))
-            self.assertEquals(b_zuul_commit, str(work_repo2.commit('HEAD')))
-
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
             shutil.rmtree(self.workspace_root)
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+    def test_upgrade(self):
+        # Simulates an upgrade test
+        self.worker.hold_jobs_in_build = True
+        projects = ['org/project1', 'org/project2', 'org/project3',
+                    'org/project4', 'org/project5', 'org/project6']
+
+        self.create_branch('org/project2', 'stable/havana')
+        self.create_branch('org/project3', 'stable/havana')
+        self.create_branch('org/project4', 'stable/havana')
+        self.create_branch('org/project5', 'stable/havana')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'C')
+        D = self.fake_gerrit.addFakeChange('org/project3', 'master', 'D')
+        E = self.fake_gerrit.addFakeChange('org/project4', 'stable/havana', 'E')
+        A.addApproval('CRVW', 2)
+        B.addApproval('CRVW', 2)
+        C.addApproval('CRVW', 2)
+        D.addApproval('CRVW', 2)
+        E.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(E.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(5, len(self.builds), "Five builds are running")
+
+        # Check the old side of the upgrade first
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+             'org/project3': str(upstream['org/project3'].commit('stable/havana')),
+             'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+             'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+             'org/project3': str(upstream['org/project3'].commit('stable/havana')),
+             'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+             'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+             'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+
+             'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+             'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+             'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+             'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project4': self.builds[4].parameters['ZUUL_COMMIT'],
+             'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            change_number = int(build.parameters['ZUUL_CHANGE'])
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                branch='stable/havana', # Old branch for upgrade
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct on old side of upgrade' %
+                                  (project, number))
+            shutil.rmtree(self.workspace_root)
+
+        # Check the new side of the upgrade
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            change_number = int(build.parameters['ZUUL_CHANGE'])
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                branch='master', # New branch for upgrade
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct on old side of upgrade' %
+                                  (project, number))
+            shutil.rmtree(self.workspace_root)
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+    def test_project_override(self):
+        self.worker.hold_jobs_in_build = True
+        projects = ['org/project1', 'org/project2', 'org/project3',
+                    'org/project4', 'org/project5', 'org/project6']
+
+        self.create_branch('org/project3', 'stable/havana')
+        self.create_branch('org/project4', 'stable/havana')
+        self.create_branch('org/project6', 'stable/havana')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+        D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'D')
+        A.addApproval('CRVW', 2)
+        B.addApproval('CRVW', 2)
+        C.addApproval('CRVW', 2)
+        D.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(4, len(self.builds), "Four builds are running")
+
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('stable/havana')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            change_number = int(build.parameters['ZUUL_CHANGE'])
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                project_branches={'org/project4': 'master'},
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
+            shutil.rmtree(self.workspace_root)
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index 1310c16..a895f24 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -54,6 +54,10 @@
         parser.add_argument('--version', dest='version', action='version',
                             version=self._get_version(),
                             help='show zuul version')
+        parser.add_argument('--cache-dir', dest='cache_dir',
+                            help=('a directory that holds cached copies of '
+                                  'repos from which to make an initial clone.'
+                                  ))
         parser.add_argument('git_base_url',
                             help='reference repo to clone from')
         parser.add_argument('projects', nargs='+',
@@ -61,17 +65,24 @@
 
         project_env = parser.add_argument_group(
             'project tuning'
-        )
+            )
         project_env.add_argument(
             '--branch',
             help=('branch to checkout instead of Zuul selected branch, '
                   'for example to specify an alternate branch to test '
                   'client library compatibility.')
-        )
+            )
+        project_env.add_argument(
+            '--project-branch', nargs=1, action='append',
+            metavar='PROJECT=BRANCH',
+            help=('project-specific branch to checkout which takes precedence '
+                  'over --branch if it is provided; may be specified multiple '
+                  'times.')
+            )
 
         zuul_env = parser.add_argument_group(
             'zuul environnement',
-            'Let you override $ZUUL_* environnement variables.'
+            'Let you override $ZUUL_* environment variables.'
         )
         for zuul_suffix in ZUUL_ENV_SUFFIXES:
             env_name = 'ZUUL_%s' % zuul_suffix.upper()
@@ -120,6 +131,11 @@
     def main(self):
         self.parse_arguments()
         self.setup_logging(color=self.args.color, verbose=self.args.verbose)
+        project_branches = {}
+        if self.args.project_branch:
+            for x in self.args.project_branch:
+                project, branch = x[0].split('=')
+                project_branches[project] = branch
         cloner = zuul.lib.cloner.Cloner(
             git_base_url=self.args.git_base_url,
             projects=self.args.projects,
@@ -128,7 +144,9 @@
             zuul_ref=self.args.zuul_ref,
             zuul_url=self.args.zuul_url,
             branch=self.args.branch,
-            clone_map_file=self.args.clone_map_file
+            clone_map_file=self.args.clone_map_file,
+            project_branches=project_branches,
+            cache_dir=self.args.cache_dir,
         )
         cloner.execute()
 
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 0961eb4..89ebada 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -28,18 +28,21 @@
     log = logging.getLogger("zuul.Cloner")
 
     def __init__(self, git_base_url, projects, workspace, zuul_branch,
-                 zuul_ref, zuul_url, branch=None, clone_map_file=None):
+                 zuul_ref, zuul_url, branch=None, clone_map_file=None,
+                 project_branches=None, cache_dir=None):
 
         self.clone_map = []
         self.dests = None
 
         self.branch = branch
         self.git_url = git_base_url
+        self.cache_dir = cache_dir
         self.projects = projects
         self.workspace = workspace
         self.zuul_branch = zuul_branch
         self.zuul_ref = zuul_ref
         self.zuul_url = zuul_url
+        self.project_branches = project_branches or {}
 
         if clone_map_file:
             self.readCloneMap(clone_map_file)
@@ -64,9 +67,24 @@
         self.log.info("Prepared all repositories")
 
     def cloneUpstream(self, project, dest):
+        # Check for a cached git repo first
+        git_cache = '%s/%s' % (self.cache_dir, project)
         git_upstream = '%s/%s' % (self.git_url, project)
-        self.log.info("Creating repo %s from upstream %s",
-                      project, git_upstream)
+        if (self.cache_dir and
+            os.path.exists(git_cache) and
+            not os.path.exists(dest)):
+            # file:// tells git not to hard-link across repos
+            git_cache = 'file://%s' % git_cache
+            self.log.info("Creating repo %s from cache %s",
+                          project, git_cache)
+            new_repo = git.Repo.clone_from(git_cache, dest)
+            self.log.info("Updating origin remote in repo %s to %s",
+                          project, git_upstream)
+            origin = new_repo.remotes.origin.config_writer.set(
+                'url', git_upstream)
+        else:
+            self.log.info("Creating repo %s from upstream %s",
+                          project, git_upstream)
         repo = Repo(
             remote=git_upstream,
             local=dest,
@@ -98,6 +116,12 @@
          2) Zuul reference for the master branch
          3) The tip of the indicated branch
          4) The tip of the master branch
+
+        The "indicated branch" is one of the following:
+
+         A) The project-specific override branch (from project_branches arg)
+         B) The user specified branch (from the branch arg)
+         C) ZUUL_BRANCH (from the zuul_branch arg)
         """
 
         repo = self.cloneUpstream(project, dest)
@@ -106,22 +130,24 @@
         # Ensure that we don't have stale remotes around
         repo.prune()
 
-        override_zuul_ref = self.zuul_ref
-        # FIXME should be origin HEAD branch which might not be 'master'
-        fallback_branch = 'master'
-        fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+        indicated_branch = self.branch or self.zuul_branch
+        if project in self.project_branches:
+            indicated_branch = self.project_branches[project]
+
+        override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
                                    self.zuul_ref)
 
-        if self.branch:
-            override_zuul_ref = re.sub(self.zuul_branch, self.branch,
-                                       self.zuul_ref)
-            if repo.hasBranch(self.branch):
-                self.log.debug("upstream repo has branch %s", self.branch)
-                fallback_branch = self.branch
-                fallback_zuul_ref = self.zuul_ref
-            else:
-                self.log.exception("upstream repo is missing branch %s",
-                                   self.branch)
+        if repo.hasBranch(indicated_branch):
+            self.log.debug("upstream repo has branch %s", indicated_branch)
+            fallback_branch = indicated_branch
+        else:
+            self.log.debug("upstream repo is missing branch %s",
+                           self.branch)
+            # FIXME should be origin HEAD branch which might not be 'master'
+            fallback_branch = 'master'
+
+        fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+                                   self.zuul_ref)
 
         if (self.fetchFromZuul(repo, project, override_zuul_ref)
             or (fallback_zuul_ref != override_zuul_ref and