Add SourceContext class
This replaces all of the instances where we were passing around
triplets of project/branch/secure. This significantly simplifies
the code.
It also ensures that every unparsed job or project config has
a source context associated with it, since it truly is required
by jobs.
Change-Id: I46fa9cc48f5ee57be0d9ad28b2f3c23a8d204d69
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index a71dc28..b7dc706 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -30,7 +30,10 @@
@property
def job(self):
layout = model.Layout()
+ project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
job = configloader.JobParser.fromYaml(layout, {
+ '_source_context': context,
'name': 'job',
'irrelevant-files': [
'^docs/.*$'
@@ -57,9 +60,10 @@
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
'nodes': [{
@@ -69,7 +73,7 @@
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'parent': 'base',
'nodes': [{
@@ -80,7 +84,7 @@
})
layout.addJob(python27)
python27diablo = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'branches': [
'stable/diablo'
@@ -94,9 +98,7 @@
layout.addJob(python27diablo)
project_config = configloader.ProjectParser.fromYaml(layout, {
- '_source_project': project,
- '_source_branch': 'master',
- '_source_configrepo': True,
+ '_source_context': context,
'name': 'project',
'gate': {
'jobs': [
@@ -146,15 +148,16 @@
def test_job_auth_inheritance(self):
layout = model.Layout()
project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
})
layout.addJob(base)
pypi_upload_without_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'pypi-upload-without-inherit',
'parent': 'base',
'timeout': 40,
@@ -166,7 +169,7 @@
})
layout.addJob(pypi_upload_without_inherit)
pypi_upload_with_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'pypi-upload-with-inherit',
'parent': 'base',
'timeout': 40,
@@ -180,7 +183,7 @@
layout.addJob(pypi_upload_with_inherit)
pypi_upload_with_inherit_false = configloader.JobParser.fromYaml(
layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'pypi-upload-with-inherit-false',
'parent': 'base',
'timeout': 40,
@@ -193,20 +196,20 @@
})
layout.addJob(pypi_upload_with_inherit_false)
in_repo_job_without_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'in-repo-job-without-inherit',
'parent': 'pypi-upload-without-inherit',
})
layout.addJob(in_repo_job_without_inherit)
in_repo_job_with_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'in-repo-job-with-inherit',
'parent': 'pypi-upload-with-inherit',
})
layout.addJob(in_repo_job_with_inherit)
in_repo_job_with_inherit_false = configloader.JobParser.fromYaml(
layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'in-repo-job-with-inherit-false',
'parent': 'pypi-upload-with-inherit-false',
})
@@ -225,22 +228,23 @@
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'parent': 'base',
'timeout': 40,
})
layout.addJob(python27)
python27diablo = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'branches': [
'stable/diablo'
@@ -250,9 +254,7 @@
layout.addJob(python27diablo)
project_config = configloader.ProjectParser.fromYaml(layout, {
- '_source_project': project,
- '_source_branch': 'master',
- '_source_configrepo': True,
+ '_source_context': context,
'name': 'project',
'gate': {
'jobs': [
@@ -298,15 +300,16 @@
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'parent': 'base',
'timeout': 40,
@@ -315,9 +318,7 @@
layout.addJob(python27)
project_config = configloader.ProjectParser.fromYaml(layout, {
- '_source_project': project,
- '_source_branch': 'master',
- '_source_configrepo': True,
+ '_source_context': context,
'name': 'project',
'gate': {
'jobs': [
@@ -342,15 +343,18 @@
def test_job_source_project(self):
layout = model.Layout()
base_project = model.Project('base_project', None)
+ base_context = model.SourceContext(base_project, 'master', True)
+
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': base_project,
+ '_source_context': base_context,
'name': 'base',
})
layout.addJob(base)
other_project = model.Project('other_project', None)
+ other_context = model.SourceContext(other_project, 'master', True)
base2 = configloader.JobParser.fromYaml(layout, {
- '_source_project': other_project,
+ '_source_context': other_context,
'name': 'base',
})
with testtools.ExpectedException(
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 8e112d0..f8bdd66 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -103,9 +103,7 @@
'attempts': int,
'pre-run': to_list(str),
'post-run': to_list(str),
- '_source_project': model.Project,
- '_source_branch': vs.Any(str, None),
- '_source_configrepo': bool,
+ '_source_context': model.SourceContext,
}
return vs.Schema(job)
@@ -145,37 +143,29 @@
# accumulate onto any previously applied tags from
# metajobs.
job.tags = job.tags.union(set(tags))
- # The source attributes and playbook info may not be
+ # The source attribute and playbook info may not be
# overridden -- they are always supplied by the config loader.
# They correspond to the Project instance of the repo where it
# originated, and the branch name.
- job.source_project = conf.get('_source_project')
- job.source_branch = conf.get('_source_branch')
- job.source_configrepo = conf.get('_source_configrepo')
+ job.source_context = conf.get('_source_context')
pre_run_name = conf.get('pre-run')
# Append the pre-run command
if pre_run_name:
pre_run_name = os.path.join('playbooks', pre_run_name)
- pre_run = model.PlaybookContext(job.source_project,
- job.source_branch,
- pre_run_name,
- job.source_configrepo)
+ pre_run = model.PlaybookContext(job.source_context,
+ pre_run_name)
job.pre_run.append(pre_run)
# Prepend the post-run command
post_run_name = conf.get('post-run')
if post_run_name:
post_run_name = os.path.join('playbooks', post_run_name)
- post_run = model.PlaybookContext(job.source_project,
- job.source_branch,
- post_run_name,
- job.source_configrepo)
+ post_run = model.PlaybookContext(job.source_context,
+ post_run_name)
job.post_run.insert(0, post_run)
# Set the run command
run_name = job.name
run_name = os.path.join('playbooks', run_name)
- run = model.PlaybookContext(job.source_project,
- job.source_branch, run_name,
- job.source_configrepo)
+ run = model.PlaybookContext(job.source_context, run_name)
job.run = run
job.failure_message = conf.get('failure-message', job.failure_message)
job.success_message = conf.get('success-message', job.success_message)
@@ -184,8 +174,8 @@
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
- if (not job.source_configrepo) and job.source_branch:
- branches = [job.source_branch]
+ if (not job.source_context.secure):
+ branches = [job.source_context.branch]
elif 'branches' in conf:
branches = as_list(conf['branches'])
else:
@@ -219,9 +209,7 @@
'merge-mode': vs.Any(
'merge', 'merge-resolve',
'cherry-pick'),
- '_source_project': model.Project,
- '_source_branch': vs.Any(str, None),
- '_source_configrepo': bool,
+ '_source_context': model.SourceContext,
}
for p in layout.pipelines.values():
@@ -233,9 +221,7 @@
def fromYaml(layout, conf):
ProjectTemplateParser.getSchema(layout)(conf)
project_template = model.ProjectConfig(conf['name'])
- source_project = conf['_source_project']
- source_branch = conf['_source_branch']
- source_configrepo = conf['_source_configrepo']
+ source_context = conf['_source_context']
for pipeline in layout.pipelines.values():
conf_pipeline = conf.get(pipeline.name)
if not conf_pipeline:
@@ -245,12 +231,11 @@
project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
layout, conf_pipeline.get('jobs', []),
- source_project, source_branch, source_configrepo)
+ source_context)
return project_template
@staticmethod
- def _parseJobTree(layout, conf, source_project, source_branch,
- source_configrepo, tree=None):
+ def _parseJobTree(layout, conf, source_context, tree=None):
if not tree:
tree = model.JobTree(None)
for conf_job in conf:
@@ -264,9 +249,7 @@
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
- attrs['_source_project'] = source_project
- attrs['_source_branch'] = source_branch
- attrs['_source_configrepo'] = source_configrepo
+ attrs['_source_context'] = source_context
subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
else:
# Not overriding, so get existing job
@@ -275,9 +258,7 @@
if jobs:
# This is the root of a sub tree
ProjectTemplateParser._parseJobTree(layout, jobs,
- source_project,
- source_branch,
- source_configrepo,
+ source_context,
subtree)
else:
raise Exception("Job must be a string or dictionary")
@@ -294,9 +275,7 @@
'templates': [str],
'merge-mode': vs.Any('merge', 'merge-resolve',
'cherry-pick'),
- '_source_project': model.Project,
- '_source_branch': vs.Any(str, None),
- '_source_configrepo': bool,
+ '_source_context': model.SourceContext,
}
for p in layout.pipelines.values():
@@ -595,9 +574,7 @@
url = source.getGitUrl(project)
job = merger.getFiles(project.name, url, 'master',
files=['zuul.yaml', '.zuul.yaml'])
- job.project = project
- job.branch = 'master'
- job.config_repo = True
+ job.source_context = model.SourceContext(project, 'master', True)
jobs.append(job)
for (source, project) in project_repos:
@@ -611,9 +588,8 @@
for branch in source.getProjectBranches(project):
job = merger.getFiles(project.name, url, branch,
files=['.zuul.yaml'])
- job.project = project
- job.branch = branch
- job.config_repo = False
+ job.source_context = model.SourceContext(project,
+ branch, False)
jobs.append(job)
for job in jobs:
@@ -627,31 +603,31 @@
if job.files.get(fn):
TenantParser.log.info(
"Loading configuration from %s/%s" %
- (job.project, fn))
- if job.config_repo:
+ (job.source_context, fn))
+ if job.source_context.secure:
incdata = TenantParser._parseConfigRepoLayout(
- job.files[fn], job.project, job.branch)
+ job.files[fn], job.source_context)
config_repos_config.extend(incdata)
else:
incdata = TenantParser._parseProjectRepoLayout(
- job.files[fn], job.project, job.branch)
+ job.files[fn], job.source_context)
project_repos_config.extend(incdata)
- job.project.unparsed_config = incdata
+ job.source_context.project.unparsed_config = incdata
return config_repos_config, project_repos_config
@staticmethod
- def _parseConfigRepoLayout(data, project, branch):
+ def _parseConfigRepoLayout(data, source_context):
# This is the top-level configuration for a tenant.
config = model.UnparsedTenantConfig()
- config.extend(yaml.load(data), project, branch, True)
+ config.extend(yaml.load(data), source_context)
return config
@staticmethod
- def _parseProjectRepoLayout(data, project, branch):
+ def _parseProjectRepoLayout(data, source_context):
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
config = model.UnparsedTenantConfig()
- config.extend(yaml.load(data), project, branch, False)
+ config.extend(yaml.load(data), source_context)
return config
@@ -722,8 +698,10 @@
data = project.unparsed_config
if not data:
continue
+ source_context = model.SourceContext(project,
+ branch, False)
incdata = TenantParser._parseProjectRepoLayout(
- data, project, branch)
+ data, source_context)
config.extend(incdata)
layout = model.Layout()
diff --git a/zuul/model.py b/zuul/model.py
index e467c00..6d6abc8 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -515,24 +515,48 @@
self.state_time = data['state_time']
+class SourceContext(object):
+ """A reference to the branch of a project in configuration.
+
+ Jobs and playbooks reference this to keep track of where they
+ originate."""
+
+ def __init__(self, project, branch, secure):
+ self.project = project
+ self.branch = branch
+ self.secure = secure
+
+ def __repr__(self):
+ return '<SourceContext %s:%s secure:%s>' % (self.project,
+ self.branch,
+ self.secure)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, SourceContext):
+ return False
+ return (self.project == other.project and
+ self.branch == other.branch and
+ self.secure == other.secure)
+
+
class PlaybookContext(object):
+
"""A reference to a playbook in the context of a project.
Jobs refer to objects of this class for their main, pre, and post
playbooks so that we can keep track of which repos and security
contexts are needed in order to run them."""
- def __init__(self, project, branch, path, secure):
- self.project = project
- self.branch = branch
+ def __init__(self, source_context, path):
+ self.source_context = source_context
self.path = path
- self.secure = secure
def __repr__(self):
- return '<PlaybookContext %s:%s %s secure:%s>' % (self.project,
- self.branch,
- self.path,
- self.secure)
+ return '<PlaybookContext %s %s>' % (self.source_context,
+ self.path)
def __ne__(self, other):
return not self.__eq__(other)
@@ -540,19 +564,17 @@
def __eq__(self, other):
if not isinstance(other, PlaybookContext):
return False
- return (self.project == other.project and
- self.branch == other.branch and
- self.path == other.path and
- self.secure == other.secure)
+ return (self.source_context == other.source_context and
+ self.path == other.path)
def toDict(self):
# Render to a dict to use in passing json to the launcher
return dict(
- connection=self.project.connection_name,
- project=self.project.name,
- branch=self.branch,
- path=self.path,
- secure=self.secure)
+ connection=self.source_context.project.connection_name,
+ project=self.source_context.project.name,
+ branch=self.source_context.branch,
+ secure=self.source_context.secure,
+ path=self.path)
class Job(object):
@@ -583,9 +605,7 @@
tags=set(),
mutex=None,
attempts=3,
- source_project=None,
- source_branch=None,
- source_configrepo=None,
+ source_context=None,
)
self.name = name
@@ -1840,8 +1860,7 @@
r.nodesets = copy.deepcopy(self.nodesets)
return r
- def extend(self, conf, source_project=None, source_branch=None,
- source_configrepo=None):
+ def extend(self, conf, source_context=None):
if isinstance(conf, UnparsedTenantConfig):
self.pipelines.extend(conf.pipelines)
self.jobs.extend(conf.jobs)
@@ -1854,6 +1873,11 @@
raise Exception("Configuration items must be in the form of "
"a list of dictionaries (when parsing %s)" %
(conf,))
+
+ if source_context is None:
+ raise Exception("A source context must be provided "
+ "(when parsing %s)" % (conf,))
+
for item in conf:
if not isinstance(item, dict):
raise Exception("Configuration items must be in the form of "
@@ -1865,12 +1889,7 @@
(conf,))
key, value = item.items()[0]
if key in ['project', 'project-template', 'job']:
- if source_project is not None:
- value['_source_project'] = source_project
- if source_branch is not None:
- value['_source_branch'] = source_branch
- if source_configrepo is not None:
- value['_source_configrepo'] = source_configrepo
+ value['_source_context'] = source_context
if key == 'project':
self.projects.append(value)
elif key == 'job':
@@ -1915,13 +1934,16 @@
def addJob(self, job):
# We can have multiple variants of a job all with the same
# name, but these variants must all be defined in the same repo.
- prior_jobs = [j for j in self.getJobs(job.name)
- if j.source_project != job.source_project]
+ prior_jobs = [j for j in self.getJobs(job.name) if
+ j.source_context.project !=
+ job.source_context.project]
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
- "job %s in %s" % (job, job.source_project,
- prior_jobs[0],
- prior_jobs[0].source_project))
+ "job %s in %s" % (
+ job,
+ job.source_context.project,
+ prior_jobs[0],
+ prior_jobs[0].source_context.project))
if job.name in self.jobs:
self.jobs[job.name].append(job)