Merge "Clean up bad layout files in zuul tests"
diff --git a/doc/source/cloner.rst b/doc/source/cloner.rst
index bb33f82..2ddf0b5 100644
--- a/doc/source/cloner.rst
+++ b/doc/source/cloner.rst
@@ -75,3 +75,15 @@
projects::
zuul-cloner project project/plugins/plugin1
+
+Cached repositories
+-------------------
+
+The ``--cache-dir`` option can be used to reduce network traffic by
+cloning from a local repository which may not be up to date.
+
+If the ``--cache-dir`` option is supplied, zuul-cloner will start by
+cloning any projects it processes from those found in that directory.
+The URL of origin remote of the resulting clone will be reset to use
+the ``git_base_url`` and then the remote will be updated so that the
+repository has all the information in the upstream repository.
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index cdfa4df..683ca00 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -978,6 +978,10 @@
check:
- foobar-extra-special-job
+Individual jobs may optionally be added to pipelines (e.g. check,
+gate, et cetera) for a project, in addtion to those provided by
+templates.
+
The order of the jobs listed in the project (which only affects the
order of jobs listed on the report) will be the jobs from each
template in the order listed, followed by any jobs individually listed
diff --git a/tests/base.py b/tests/base.py
index 179f4f4..46c7087 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -56,6 +56,7 @@
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'fixtures')
+USE_TEMPDIR = True
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-32s '
@@ -165,7 +166,7 @@
if files:
fn = files[0]
else:
- fn = '%s-%s' % (self.branch, self.number)
+ fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
msg = self.subject + '-' + str(self.latest_patchset)
c = self.add_fake_change_to_repo(msg, fn, large)
ps_files = [{'file': '/COMMIT_MSG',
@@ -821,8 +822,11 @@
level=logging.DEBUG,
format='%(asctime)s %(name)-32s '
'%(levelname)-8s %(message)s'))
- tmp_root = self.useFixture(fixtures.TempDir(
- rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+ if USE_TEMPDIR:
+ tmp_root = self.useFixture(fixtures.TempDir(
+ rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+ else:
+ tmp_root = os.environ.get("ZUUL_TEST_ROOT")
self.test_root = os.path.join(tmp_root, "zuul-test")
self.upstream_root = os.path.join(self.test_root, "upstream")
self.git_root = os.path.join(self.test_root, "git")
@@ -844,6 +848,9 @@
self.init_repo("org/project1")
self.init_repo("org/project2")
self.init_repo("org/project3")
+ self.init_repo("org/project4")
+ self.init_repo("org/project5")
+ self.init_repo("org/project6")
self.init_repo("org/one-job-project")
self.init_repo("org/nonvoting-project")
self.init_repo("org/templated-project")
@@ -976,7 +983,6 @@
threads = threading.enumerate()
if len(threads) > 1:
self.log.error("More than one thread is running: %s" % threads)
- super(ZuulTestCase, self).tearDown()
def init_repo(self, project):
parts = project.split('/')
@@ -999,15 +1005,26 @@
master = repo.create_head('master')
repo.create_tag('init')
- mp = repo.create_head('mp')
- repo.head.reference = mp
+ repo.head.reference = master
+ repo.head.reset(index=True, working_tree=True)
+ repo.git.clean('-x', '-f', '-d')
+
+ self.create_branch(project, 'mp')
+
+ def create_branch(self, project, branch):
+ path = os.path.join(self.upstream_root, project)
+ repo = git.Repo.init(path)
+ fn = os.path.join(path, 'README')
+
+ branch_head = repo.create_head(branch)
+ repo.head.reference = branch_head
f = open(fn, 'a')
- f.write("test mp\n")
+ f.write("test %s\n" % branch)
f.close()
repo.index.add([fn])
- repo.index.commit('mp commit')
+ repo.index.commit('%s commit' % branch)
- repo.head.reference = master
+ repo.head.reference = repo.heads['master']
repo.head.reset(index=True, working_tree=True)
repo.git.clean('-x', '-f', '-d')
diff --git a/tests/fixtures/layout-gating.yaml b/tests/fixtures/layout-cloner.yaml
similarity index 65%
rename from tests/fixtures/layout-gating.yaml
rename to tests/fixtures/layout-cloner.yaml
index a544a80..e840ed9 100644
--- a/tests/fixtures/layout-gating.yaml
+++ b/tests/fixtures/layout-cloner.yaml
@@ -22,8 +22,24 @@
- name: org/project1
gate:
- - project1-project2-integration
+ - integration
- name: org/project2
gate:
- - project1-project2-integration
+ - integration
+
+ - name: org/project3
+ gate:
+ - integration
+
+ - name: org/project4
+ gate:
+ - integration
+
+ - name: org/project5
+ gate:
+ - integration
+
+ - name: org/project6
+ gate:
+ - integration
diff --git a/tests/test_cloner.py b/tests/test_cloner.py
index bb9d91f..ab2683d 100644
--- a/tests/test_cloner.py
+++ b/tests/test_cloner.py
@@ -41,21 +41,88 @@
self.workspace_root = os.path.join(self.test_root, 'workspace')
self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-gating.yaml')
+ 'tests/fixtures/layout-cloner.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
- def test_cloner(self):
+ def getWorkspaceRepos(self, projects):
+ repos = {}
+ for project in projects:
+ repos[project] = git.Repo(
+ os.path.join(self.workspace_root, project))
+ return repos
+
+ def getUpstreamRepos(self, projects):
+ repos = {}
+ for project in projects:
+ repos[project] = git.Repo(
+ os.path.join(self.upstream_root, project))
+ return repos
+
+ def test_cache_dir(self):
+ projects = ['org/project1', 'org/project2']
+ cache_root = os.path.join(self.test_root, "cache")
+ for project in projects:
+ upstream_repo_path = os.path.join(self.upstream_root, project)
+ cache_repo_path = os.path.join(cache_root, project)
+ git.Repo.clone_from(upstream_repo_path, cache_repo_path)
+
+ self.worker.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(1, len(self.builds), "One build is running")
+
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ B.setMerged()
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.upstream_root,
+ projects=projects,
+ workspace=self.workspace_root,
+ zuul_branch=build.parameters['ZUUL_BRANCH'],
+ zuul_ref=build.parameters['ZUUL_REF'],
+ zuul_url=self.git_root,
+ cache_dir=cache_root,
+ )
+ cloner.execute()
+ work = self.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertEquals(state[project],
+ str(work[project].commit('HEAD')),
+ 'Project %s commit for build %s should '
+ 'be correct' % (project, number))
+
+ work = self.getWorkspaceRepos(projects)
+ upstream_repo_path = os.path.join(self.upstream_root, 'org/project1')
+ self.assertEquals(work['org/project1'].remotes.origin.url,
+ upstream_repo_path,
+ 'workspace repo origin should be upstream, not cache')
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ def test_one_branch(self):
self.worker.hold_jobs_in_build = True
+ projects = ['org/project1', 'org/project2']
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
-
- A.addPatchset(['project_one.txt'])
- B.addPatchset(['project_two.txt'])
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
-
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
@@ -65,39 +132,352 @@
self.assertEquals(2, len(self.builds), "Two builds are running")
- a_zuul_ref = b_zuul_ref = None
- for build in self.builds:
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
- if build.parameters['ZUUL_CHANGE'] == '1':
- a_zuul_ref = build.parameters['ZUUL_REF']
- a_zuul_commit = build.parameters['ZUUL_COMMIT']
- if build.parameters['ZUUL_CHANGE'] == '2':
- b_zuul_ref = build.parameters['ZUUL_REF']
- b_zuul_commit = build.parameters['ZUUL_COMMIT']
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.upstream_root,
+ projects=projects,
+ workspace=self.workspace_root,
+ zuul_branch=build.parameters['ZUUL_BRANCH'],
+ zuul_ref=build.parameters['ZUUL_REF'],
+ zuul_url=self.git_root,
+ )
+ cloner.execute()
+ work = self.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertEquals(state[project],
+ str(work[project].commit('HEAD')),
+ 'Project %s commit for build %s should '
+ 'be correct' % (project, number))
+
+ shutil.rmtree(self.workspace_root)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
- # Repos setup, now test the cloner
- for zuul_ref in [a_zuul_ref, b_zuul_ref]:
+ def test_multi_branch(self):
+ self.worker.hold_jobs_in_build = True
+ projects = ['org/project1', 'org/project2',
+ 'org/project3', 'org/project4']
+
+ self.create_branch('org/project2', 'stable/havana')
+ self.create_branch('org/project4', 'stable/havana')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+ A.addApproval('CRVW', 2)
+ B.addApproval('CRVW', 2)
+ C.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(3, len(self.builds), "Three builds are running")
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].
+ commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].
+ commit('stable/havana')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+ 'org/project4': str(upstream['org/project4'].
+ commit('master')),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
- projects=['org/project1', 'org/project2'],
+ projects=projects,
workspace=self.workspace_root,
- zuul_branch='master',
- zuul_ref=zuul_ref,
+ zuul_branch=build.parameters['ZUUL_BRANCH'],
+ zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
- branch='master',
- clone_map_file=os.path.join(FIXTURE_DIR, 'clonemap.yaml')
- )
+ )
cloner.execute()
- work_repo1 = git.Repo(os.path.join(self.workspace_root,
- 'org/project1'))
- self.assertEquals(a_zuul_commit, str(work_repo1.commit('HEAD')))
+ work = self.getWorkspaceRepos(projects)
+ state = states[number]
- work_repo2 = git.Repo(os.path.join(self.workspace_root,
- 'org/project2'))
- self.assertEquals(b_zuul_commit, str(work_repo2.commit('HEAD')))
-
+ for project in projects:
+ self.assertEquals(state[project],
+ str(work[project].commit('HEAD')),
+ 'Project %s commit for build %s should '
+ 'be correct' % (project, number))
shutil.rmtree(self.workspace_root)
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ def test_upgrade(self):
+ # Simulates an upgrade test
+ self.worker.hold_jobs_in_build = True
+ projects = ['org/project1', 'org/project2', 'org/project3',
+ 'org/project4', 'org/project5', 'org/project6']
+
+ self.create_branch('org/project2', 'stable/havana')
+ self.create_branch('org/project3', 'stable/havana')
+ self.create_branch('org/project4', 'stable/havana')
+ self.create_branch('org/project5', 'stable/havana')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'C')
+ D = self.fake_gerrit.addFakeChange('org/project3', 'master', 'D')
+ E = self.fake_gerrit.addFakeChange('org/project4', 'stable/havana', 'E')
+ A.addApproval('CRVW', 2)
+ B.addApproval('CRVW', 2)
+ C.addApproval('CRVW', 2)
+ D.addApproval('CRVW', 2)
+ E.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(E.addApproval('APRV', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(5, len(self.builds), "Five builds are running")
+
+ # Check the old side of the upgrade first
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+ 'org/project3': str(upstream['org/project3'].commit('stable/havana')),
+ 'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+ 'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+ 'org/project3': str(upstream['org/project3'].commit('stable/havana')),
+ 'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+ 'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+ 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+ 'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+
+ 'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+ 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+ 'org/project4': str(upstream['org/project4'].commit('stable/havana')),
+ 'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('stable/havana')),
+ 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
+ 'org/project4': self.builds[4].parameters['ZUUL_COMMIT'],
+ 'org/project5': str(upstream['org/project5'].commit('stable/havana')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ change_number = int(build.parameters['ZUUL_CHANGE'])
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.upstream_root,
+ projects=projects,
+ workspace=self.workspace_root,
+ zuul_branch=build.parameters['ZUUL_BRANCH'],
+ zuul_ref=build.parameters['ZUUL_REF'],
+ zuul_url=self.git_root,
+ branch='stable/havana', # Old branch for upgrade
+ )
+ cloner.execute()
+ work = self.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertEquals(state[project],
+ str(work[project].commit('HEAD')),
+ 'Project %s commit for build %s should '
+ 'be correct on old side of upgrade' %
+ (project, number))
+ shutil.rmtree(self.workspace_root)
+
+ # Check the new side of the upgrade
+ states = [
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ change_number = int(build.parameters['ZUUL_CHANGE'])
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.upstream_root,
+ projects=projects,
+ workspace=self.workspace_root,
+ zuul_branch=build.parameters['ZUUL_BRANCH'],
+ zuul_ref=build.parameters['ZUUL_REF'],
+ zuul_url=self.git_root,
+ branch='master', # New branch for upgrade
+ )
+ cloner.execute()
+ work = self.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertEquals(state[project],
+ str(work[project].commit('HEAD')),
+ 'Project %s commit for build %s should '
+ 'be correct on old side of upgrade' %
+ (project, number))
+ shutil.rmtree(self.workspace_root)
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ def test_project_override(self):
+ self.worker.hold_jobs_in_build = True
+ projects = ['org/project1', 'org/project2', 'org/project3',
+ 'org/project4', 'org/project5', 'org/project6']
+
+ self.create_branch('org/project3', 'stable/havana')
+ self.create_branch('org/project4', 'stable/havana')
+ self.create_branch('org/project6', 'stable/havana')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+ D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'D')
+ A.addApproval('CRVW', 2)
+ B.addApproval('CRVW', 2)
+ C.addApproval('CRVW', 2)
+ D.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(4, len(self.builds), "Four builds are running")
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project2': str(upstream['org/project2'].commit('master')),
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+ 'org/project3': str(upstream['org/project3'].commit('master')),
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('master')),
+ },
+ {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+ 'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+ 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+ 'org/project4': str(upstream['org/project4'].commit('master')),
+ 'org/project5': str(upstream['org/project5'].commit('master')),
+ 'org/project6': str(upstream['org/project6'].commit('stable/havana')),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ change_number = int(build.parameters['ZUUL_CHANGE'])
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.upstream_root,
+ projects=projects,
+ workspace=self.workspace_root,
+ zuul_branch=build.parameters['ZUUL_BRANCH'],
+ zuul_ref=build.parameters['ZUUL_REF'],
+ zuul_url=self.git_root,
+ project_branches={'org/project4': 'master'},
+ )
+ cloner.execute()
+ work = self.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertEquals(state[project],
+ str(work[project].commit('HEAD')),
+ 'Project %s commit for build %s should '
+ 'be correct' % (project, number))
+ shutil.rmtree(self.workspace_root)
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
diff --git a/tests/test_zuultrigger.py b/tests/test_zuultrigger.py
index 9e9bc61..9a90a98 100644
--- a/tests/test_zuultrigger.py
+++ b/tests/test_zuultrigger.py
@@ -84,11 +84,17 @@
# A, B, C; B conflicts with A, but C does not.
# When A is merged, B and C should be checked for conflicts,
# and B should receive a -1.
+ # D and E are used to repeat the test in the second part, but
+ # are defined here to that they end up in the trigger cache.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
+ D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
+ E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
A.addPatchset(['conflict'])
B.addPatchset(['conflict'])
+ D.addPatchset(['conflict2'])
+ E.addPatchset(['conflict2'])
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
@@ -98,8 +104,33 @@
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 1)
self.assertEqual(C.reported, 0)
+ self.assertEqual(D.reported, 0)
+ self.assertEqual(E.reported, 0)
self.assertEqual(B.messages[0],
"Merge Failed.\n\nThis change was unable to be automatically "
"merged with the current state of the repository. Please rebase "
"your change and upload a new patchset.")
self.assertEqual(self.fake_gerrit.queries[0], "project:org/project status:open")
+
+ # Reconfigure and run the test again. This is a regression
+ # check to make sure that we don't end up with a stale trigger
+ # cache that has references to projects from the old
+ # configuration.
+ self.sched.reconfigure(self.config)
+
+ D.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.history), 2)
+ self.assertEqual(self.history[1].name, 'project-gate')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 1)
+ self.assertEqual(C.reported, 0)
+ self.assertEqual(D.reported, 2)
+ self.assertEqual(E.reported, 1)
+ self.assertEqual(E.messages[0],
+ "Merge Failed.\n\nThis change was unable to be automatically "
+ "merged with the current state of the repository. Please rebase "
+ "your change and upload a new patchset.")
+ self.assertEqual(self.fake_gerrit.queries[1], "project:org/project status:open")
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index 1310c16..a895f24 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -54,6 +54,10 @@
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
+ parser.add_argument('--cache-dir', dest='cache_dir',
+ help=('a directory that holds cached copies of '
+ 'repos from which to make an initial clone.'
+ ))
parser.add_argument('git_base_url',
help='reference repo to clone from')
parser.add_argument('projects', nargs='+',
@@ -61,17 +65,24 @@
project_env = parser.add_argument_group(
'project tuning'
- )
+ )
project_env.add_argument(
'--branch',
help=('branch to checkout instead of Zuul selected branch, '
'for example to specify an alternate branch to test '
'client library compatibility.')
- )
+ )
+ project_env.add_argument(
+ '--project-branch', nargs=1, action='append',
+ metavar='PROJECT=BRANCH',
+ help=('project-specific branch to checkout which takes precedence '
+ 'over --branch if it is provided; may be specified multiple '
+ 'times.')
+ )
zuul_env = parser.add_argument_group(
'zuul environnement',
- 'Let you override $ZUUL_* environnement variables.'
+ 'Let you override $ZUUL_* environment variables.'
)
for zuul_suffix in ZUUL_ENV_SUFFIXES:
env_name = 'ZUUL_%s' % zuul_suffix.upper()
@@ -120,6 +131,11 @@
def main(self):
self.parse_arguments()
self.setup_logging(color=self.args.color, verbose=self.args.verbose)
+ project_branches = {}
+ if self.args.project_branch:
+ for x in self.args.project_branch:
+ project, branch = x[0].split('=')
+ project_branches[project] = branch
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.args.git_base_url,
projects=self.args.projects,
@@ -128,7 +144,9 @@
zuul_ref=self.args.zuul_ref,
zuul_url=self.args.zuul_url,
branch=self.args.branch,
- clone_map_file=self.args.clone_map_file
+ clone_map_file=self.args.clone_map_file,
+ project_branches=project_branches,
+ cache_dir=self.args.cache_dir,
)
cloner.execute()
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 0961eb4..89ebada 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -28,18 +28,21 @@
log = logging.getLogger("zuul.Cloner")
def __init__(self, git_base_url, projects, workspace, zuul_branch,
- zuul_ref, zuul_url, branch=None, clone_map_file=None):
+ zuul_ref, zuul_url, branch=None, clone_map_file=None,
+ project_branches=None, cache_dir=None):
self.clone_map = []
self.dests = None
self.branch = branch
self.git_url = git_base_url
+ self.cache_dir = cache_dir
self.projects = projects
self.workspace = workspace
self.zuul_branch = zuul_branch
self.zuul_ref = zuul_ref
self.zuul_url = zuul_url
+ self.project_branches = project_branches or {}
if clone_map_file:
self.readCloneMap(clone_map_file)
@@ -64,9 +67,24 @@
self.log.info("Prepared all repositories")
def cloneUpstream(self, project, dest):
+ # Check for a cached git repo first
+ git_cache = '%s/%s' % (self.cache_dir, project)
git_upstream = '%s/%s' % (self.git_url, project)
- self.log.info("Creating repo %s from upstream %s",
- project, git_upstream)
+ if (self.cache_dir and
+ os.path.exists(git_cache) and
+ not os.path.exists(dest)):
+ # file:// tells git not to hard-link across repos
+ git_cache = 'file://%s' % git_cache
+ self.log.info("Creating repo %s from cache %s",
+ project, git_cache)
+ new_repo = git.Repo.clone_from(git_cache, dest)
+ self.log.info("Updating origin remote in repo %s to %s",
+ project, git_upstream)
+ origin = new_repo.remotes.origin.config_writer.set(
+ 'url', git_upstream)
+ else:
+ self.log.info("Creating repo %s from upstream %s",
+ project, git_upstream)
repo = Repo(
remote=git_upstream,
local=dest,
@@ -98,6 +116,12 @@
2) Zuul reference for the master branch
3) The tip of the indicated branch
4) The tip of the master branch
+
+ The "indicated branch" is one of the following:
+
+ A) The project-specific override branch (from project_branches arg)
+ B) The user specified branch (from the branch arg)
+ C) ZUUL_BRANCH (from the zuul_branch arg)
"""
repo = self.cloneUpstream(project, dest)
@@ -106,22 +130,24 @@
# Ensure that we don't have stale remotes around
repo.prune()
- override_zuul_ref = self.zuul_ref
- # FIXME should be origin HEAD branch which might not be 'master'
- fallback_branch = 'master'
- fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+ indicated_branch = self.branch or self.zuul_branch
+ if project in self.project_branches:
+ indicated_branch = self.project_branches[project]
+
+ override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
self.zuul_ref)
- if self.branch:
- override_zuul_ref = re.sub(self.zuul_branch, self.branch,
- self.zuul_ref)
- if repo.hasBranch(self.branch):
- self.log.debug("upstream repo has branch %s", self.branch)
- fallback_branch = self.branch
- fallback_zuul_ref = self.zuul_ref
- else:
- self.log.exception("upstream repo is missing branch %s",
- self.branch)
+ if repo.hasBranch(indicated_branch):
+ self.log.debug("upstream repo has branch %s", indicated_branch)
+ fallback_branch = indicated_branch
+ else:
+ self.log.debug("upstream repo is missing branch %s",
+ self.branch)
+ # FIXME should be origin HEAD branch which might not be 'master'
+ fallback_branch = 'master'
+
+ fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+ self.zuul_ref)
if (self.fetchFromZuul(repo, project, override_zuul_ref)
or (fallback_zuul_ref != override_zuul_ref and
diff --git a/zuul/model.py b/zuul/model.py
index 77ab68b..1a99650 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -589,6 +589,12 @@
PENDING = 2
COMPLETE = 3
+ states_map = {
+ 1: 'NEW',
+ 2: 'PENDING',
+ 3: 'COMPLETE',
+ }
+
def __init__(self, item):
self.item = item
self.other_changes = []
@@ -603,6 +609,12 @@
self.failing_reasons = []
self.merge_state = self.NEW
+ def __repr__(self):
+ return '<BuildSet item: %s #builds: %s merge state: %s>' % (
+ self.item,
+ len(self.builds),
+ self.getStateName(self.merge_state))
+
def setConfiguration(self):
# The change isn't enqueued until after it's created
# so we don't know what the other changes ahead will be
@@ -615,6 +627,10 @@
if not self.ref:
self.ref = 'Z' + uuid4().hex
+ def getStateName(self, state_num):
+ return self.states_map.get(
+ state_num, 'UNKNOWN (%s)' % state_num)
+
def addBuild(self, build):
self.builds[build.job.name] = build
build.build_set = self
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a2e07cd..d6c51e2 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -183,7 +183,6 @@
self.triggers = dict()
self.reporters = dict()
self.config = None
- self._maintain_trigger_cache = False
self.trigger_event_queue = Queue.Queue()
self.result_event_queue = Queue.Queue()
@@ -667,6 +666,7 @@
"Exception while canceling build %s "
"for change %s" % (build, item.change))
self.layout = layout
+ self.maintainTriggerCache()
for trigger in self.triggers.values():
trigger.postConfig()
if statsd:
@@ -784,10 +784,6 @@
while pipeline.manager.processQueue():
pass
- if self._maintain_trigger_cache:
- self.maintainTriggerCache()
- self._maintain_trigger_cache = False
-
except Exception:
self.log.exception("Exception in run handler:")
# There may still be more events to process
@@ -1171,7 +1167,6 @@
self.log.debug("Removing change %s from queue" % item.change)
change_queue = self.pipeline.getQueue(item.change.project)
change_queue.dequeueItem(item)
- self.sched._maintain_trigger_cache = True
def removeChange(self, change):
# Remove a change from the queue, probably because it has been
diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py
index 4d4deb8..6966488 100644
--- a/zuul/trigger/gerrit.py
+++ b/zuul/trigger/gerrit.py
@@ -280,8 +280,12 @@
# This lets the user supply a list of change objects that are
# still in use. Anything in our cache that isn't in the supplied
# list should be safe to remove from the cache.
- # TODO(jeblair): consider removing this feature
- return
+ remove = []
+ for key, change in self._change_cache.items():
+ if change not in relevant:
+ remove.append(key)
+ for key in remove:
+ del self._change_cache[key]
def postConfig(self):
pass