Use override-checkout to select jobs
If a job is defined in a project with "master" and "stable" branches,
and a user wants to inherit from that job in a project with only a
"devel" branch, Zuul will not run the job because there are no matching
variants. To correct this, have Zuul use the value of override-checkout
to mask the branch of the item under test so that it appears to match the
'override-checkout' branch.
Change-Id: I4619a6f9fe9c3203a7285e5eb4f74c52b650c737
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 525cb38..f28601b 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -692,6 +692,11 @@
attribute to apply this behavior to a subset of a job's
projects.
+ This value is also used to help select which variants of a job
+ to run. If ``override-checkout`` is set, then Zuul will use
+ this value instead of the branch of the item being tested when
+ collecting jobs to run.
+
.. attr:: timeout
The time in seconds that the job should be allowed to run before
@@ -837,6 +842,12 @@
:attr:`job.override-checkout` attribute to apply the same
behavior to all projects in a job.
+ This value is also used to help select which variants of a
+ job to run. If ``override-checkout`` is set, then Zuul will
+ use this value instead of the branch of the item being tested
+ when collecting any jobs to run which are defined in this
+ project.
+
.. attr:: vars
A dictionary of variables to supply to Ansible. When inheriting
@@ -895,6 +906,12 @@
branch of an item, then that job is not run for the item.
Otherwise, all of the job variants which match that branch (and
any other selection criteria) are used when freezing the job.
+ However, if :attr:`job.override-checkout` or
+ :attr:`job.required-projects.override-checkout` are set for a
+ project, Zuul will attempt to use the job variants which match
+ the values supplied in ``override-checkout`` for jobs defined in
+ those projects. This can be used to run a job defined in one
+ project on another project without a matching branch.
This example illustrates a job called *run-tests* which uses a
nodeset based on the current release of an operating system to
diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml
new file mode 100644
index 0000000..9954846
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml
@@ -0,0 +1,22 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ parent: null
+ run: playbooks/base.yaml
+
+- project:
+ name: common-config
+ check:
+ jobs: []
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/README b/tests/fixtures/config/branch-mismatch/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..809f830
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml
@@ -0,0 +1,7 @@
+- job:
+ name: project-test1
+
+- project:
+ check:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/README b/tests/fixtures/config/branch-mismatch/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..3a8e9df
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml
@@ -0,0 +1,13 @@
+- job:
+ name: project-test2
+ parent: project-test1
+ override-checkout: stable
+
+- project:
+ check:
+ jobs:
+ - project-test1:
+ required-projects:
+ - name: org/project1
+ override-checkout: stable
+ - project-test2
diff --git a/tests/fixtures/config/branch-mismatch/main.yaml b/tests/fixtures/config/branch-mismatch/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/branch-mismatch/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 163a58b..164d5b4 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -497,6 +497,42 @@
self.waitUntilSettled()
+class TestBranchMismatch(ZuulTestCase):
+ tenant_config_file = 'config/branch-mismatch/main.yaml'
+
+ def test_job_override_branch(self):
+ "Test that override-checkout overrides branch matchers as well"
+
+ # Make sure the parent job repo is branched, so it gets
+ # implied branch matchers.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+
+ # The child job repo should have a branch which does not exist
+ # in the parent job repo.
+ self.create_branch('org/project2', 'devel')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project2', 'devel'))
+
+ # A job in a repo with a weird branch name should use the
+ # parent job from the parent job's master (default) branch.
+ A = self.fake_gerrit.addFakeChange('org/project2', 'devel', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ # project-test2 should run because it inherits from
+ # project-test1 and we will use the fallback branch to find
+ # project-test1 variants, but project-test1 itself, even
+ # though it is in the project-pipeline config, should not run
+ # because it doesn't directly match.
+ self.assertHistory([
+ dict(name='project-test1', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
class TestCentralJobs(ZuulTestCase):
tenant_config_file = 'config/central-jobs/main.yaml'
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d622370..43bfbc5 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -700,10 +700,10 @@
(trusted, project) = tenant.getProject(project_name)
if project is None:
raise Exception("Unknown project %s" % (project_name,))
- job_project = model.JobProject(project_name,
+ job_project = model.JobProject(project.canonical_name,
project_override_branch,
project_override_checkout)
- new_projects[project_name] = job_project
+ new_projects[project.canonical_name] = job_project
job.required_projects = new_projects
tags = conf.get('tags')
diff --git a/zuul/model.py b/zuul/model.py
index 29c5a9d..a6d2bcc 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1105,8 +1105,18 @@
self.inheritance_path = self.inheritance_path + (repr(other),)
- def changeMatches(self, change):
- if self.branch_matcher and not self.branch_matcher.matches(change):
+ def changeMatches(self, change, override_branch=None):
+ if override_branch is None:
+ branch_change = change
+ else:
+ # If an override branch is supplied, create a very basic
+ # change (a Ref) and set its branch to the override
+ # branch.
+ branch_change = Ref(change.project)
+ branch_change.ref = override_branch
+
+ if self.branch_matcher and not self.branch_matcher.matches(
+ branch_change):
return False
if self.file_matcher and not self.file_matcher.matches(change):
@@ -2068,9 +2078,6 @@
def isUpdateOf(self, other):
return False
- def filterJobs(self, jobs):
- return filter(lambda job: job.changeMatches(self), jobs)
-
def getRelatedChanges(self):
return set()
@@ -2618,21 +2625,33 @@
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
- def collectJobs(self, item, jobname, change, path=None, jobs=None,
- stack=None):
- if stack is None:
- stack = []
- if jobs is None:
- jobs = []
- if path is None:
- path = []
- path.append(jobname)
+ def _updateOverrideCheckouts(self, override_checkouts, job):
+ # Update the values in an override_checkouts dict with those
+ # in a job. Used in collectJobVariants.
+ if job.override_checkout:
+ override_checkouts[None] = job.override_checkout
+ for req in job.required_projects.values():
+ if req.override_checkout:
+ override_checkouts[req.project_name] = req.override_checkout
+
+ def _collectJobVariants(self, item, jobname, change, path, jobs, stack,
+ override_checkouts, indent):
matched = False
- indent = len(path) + 1
- item.debug("Collecting job variants for {jobname}".format(
- jobname=jobname), indent=indent)
+ local_override_checkouts = override_checkouts.copy()
+ override_branch = None
+ project = None
for variant in self.getJobs(jobname):
- if not variant.changeMatches(change):
+ if project is None and variant.source_context:
+ project = variant.source_context.project
+ if override_checkouts.get(None) is not None:
+ override_branch = override_checkouts.get(None)
+ override_branch = override_checkouts.get(
+ project.canonical_name, override_branch)
+ branches = self.tenant.getProjectBranches(project)
+ if override_branch not in branches:
+ override_branch = None
+ if not variant.changeMatches(change,
+ override_branch=override_branch):
self.log.debug("Variant %s did not match %s", repr(variant),
change)
item.debug("Variant {variant} did not match".format(
@@ -2648,17 +2667,53 @@
parent = self.tenant.default_base_job
else:
parent = None
+ self._updateOverrideCheckouts(local_override_checkouts, variant)
if parent and parent not in path:
if parent in stack:
raise Exception("Dependency cycle in jobs: %s" % stack)
self.collectJobs(item, parent, change, path, jobs,
- stack + [jobname])
+ stack + [jobname], local_override_checkouts)
matched = True
- jobs.append(variant)
+ if variant not in jobs:
+ jobs.append(variant)
+ return matched
+
+ def collectJobs(self, item, jobname, change, path=None, jobs=None,
+ stack=None, override_checkouts=None):
+ # Stack is the recursion stack of job parent names. Each time
+ # we go up a level, we add to stack, and it's popped as we
+ # descend.
+ if stack is None:
+ stack = []
+ # Jobs is the list of jobs we've accumulated.
+ if jobs is None:
+ jobs = []
+ # Path is the list of job names we've examined. It
+ # accumulates and never reduces. If more than one job has the
+ # same parent, this will prevent us from adding it a second
+ # time.
+ if path is None:
+ path = []
+ # Override_checkouts is a dictionary of canonical project
+ # names -> branch names. It is not mutated, but instead new
+ # copies are made and updated as we ascend the hierarchy, so
+ # higher levels don't affect lower levels after we descend.
+ # It's used to override the branch matchers for jobs.
+ if override_checkouts is None:
+ override_checkouts = {}
+ path.append(jobname)
+ matched = False
+ indent = len(path) + 1
+ msg = "Collecting job variants for {jobname}".format(jobname=jobname)
+ self.log.debug(msg)
+ item.debug(msg, indent=indent)
+ matched = self._collectJobVariants(
+ item, jobname, change, path, jobs, stack, override_checkouts,
+ indent)
if not matched:
self.log.debug("No matching parents for job %s and change %s",
jobname, change)
- item.debug("No matching parent for {jobname}".format(
+ item.debug("No matching parents for {jobname}".format(
jobname=repr(jobname)), indent=indent)
raise NoMatchingParentError()
return jobs
@@ -2673,8 +2728,17 @@
self.log.debug("Collecting jobs %s for %s", jobname, change)
item.debug("Freezing job {jobname}".format(
jobname=jobname), indent=1)
+ # Create the initial list of override_checkouts, which are
+ # used as we walk up the hierarchy to expand the set of
+ # jobs which match.
+ override_checkouts = {}
+ for variant in job_list.jobs[jobname]:
+ if variant.changeMatches(change):
+ self._updateOverrideCheckouts(override_checkouts, variant)
try:
- variants = self.collectJobs(item, jobname, change)
+ variants = self.collectJobs(
+ item, jobname, change,
+ override_checkouts=override_checkouts)
except NoMatchingParentError:
variants = None
if not variants: