Speed configuration building
Together, these changes build an OpenStack-sized configuration in
8% of the time it currently takes.
Change-Id: I85f538a7ebdb82724559203e2c5d5380c07f07e7
diff --git a/tests/fixtures/layouts/job-vars.yaml b/tests/fixtures/layouts/job-vars.yaml
new file mode 100644
index 0000000..22fc5c2
--- /dev/null
+++ b/tests/fixtures/layouts/job-vars.yaml
@@ -0,0 +1,75 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ parent: null
+
+- job:
+ name: parentjob
+ parent: base
+ required-projects:
+ - org/project0
+ vars:
+ override: 0
+ child1override: 0
+ parent: 0
+
+- job:
+ name: child1
+ parent: parentjob
+ required-projects:
+ - org/project1
+ vars:
+ override: 1
+ child1override: 1
+ child1: 1
+
+- job:
+ name: child2
+ parent: parentjob
+ required-projects:
+ - org/project2
+ vars:
+ override: 2
+ child2: 2
+
+- job:
+ name: child3
+ parent: parentjob
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - parentjob
+ - child1
+ - child2
+ - child3:
+ required-projects:
+ - org/project3
+ vars:
+ override: 3
+ child3: 3
+
+- project:
+ name: org/project0
+
+- project:
+ name: org/project1
+
+- project:
+ name: org/project2
+
+- project:
+ name: org/project3
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index e368108..628a45c 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -246,7 +246,11 @@
})
layout.addJob(python27essex)
- project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
+ project_template_parser = configloader.ProjectTemplateParser(
+ tenant, layout)
+ project_parser = configloader.ProjectParser(
+ tenant, layout, project_template_parser)
+ project_config = project_parser.fromYaml([{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@@ -505,6 +509,7 @@
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout(tenant)
+
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@@ -539,7 +544,11 @@
})
layout.addJob(python27diablo)
- project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
+ project_template_parser = configloader.ProjectTemplateParser(
+ tenant, layout)
+ project_parser = configloader.ProjectParser(
+ tenant, layout, project_template_parser)
+ project_config = project_parser.fromYaml([{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@@ -609,7 +618,11 @@
})
layout.addJob(python27)
- project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
+ project_template_parser = configloader.ProjectTemplateParser(
+ tenant, layout)
+ project_parser = configloader.ProjectParser(
+ tenant, layout, project_template_parser)
+ project_config = project_parser.fromYaml([{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@@ -682,8 +695,12 @@
context2 = model.SourceContext(project2, 'master',
'test', True)
- project2_config = configloader.ProjectParser.fromYaml(
- self.tenant, self.layout, [{
+ project_template_parser = configloader.ProjectTemplateParser(
+ self.tenant, self.layout)
+ project_parser = configloader.ProjectParser(
+ self.tenant, self.layout, project_template_parser)
+ project2_config = project_parser.fromYaml(
+ [{
'_source_context': context2,
'_start_mark': self.start_mark,
'name': 'project2',
@@ -718,8 +735,12 @@
self.layout.addJob(job)
- project_config = configloader.ProjectParser.fromYaml(
- self.tenant, self.layout, [{
+ project_template_parser = configloader.ProjectTemplateParser(
+ self.tenant, self.layout)
+ project_parser = configloader.ProjectParser(
+ self.tenant, self.layout, project_template_parser)
+ project_config = project_parser.fromYaml(
+ [{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 5226675..65a37ff 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2269,6 +2269,58 @@
self.assertEqual(set(['project-test-nomatch-starts-empty',
'project-test-nomatch-starts-full']), run_jobs)
+ @simple_layout('layouts/job-vars.yaml')
+ def test_inherited_job_variables(self):
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='parentjob', result='SUCCESS'),
+ dict(name='child1', result='SUCCESS'),
+ dict(name='child2', result='SUCCESS'),
+ dict(name='child3', result='SUCCESS'),
+ ], ordered=False)
+ j = self.getJobFromHistory('parentjob')
+ rp = set([p['name'] for p in j.parameters['projects']])
+ self.assertEqual(j.parameters['vars']['override'], 0)
+ self.assertEqual(j.parameters['vars']['child1override'], 0)
+ self.assertEqual(j.parameters['vars']['parent'], 0)
+ self.assertFalse('child1' in j.parameters['vars'])
+ self.assertFalse('child2' in j.parameters['vars'])
+ self.assertFalse('child3' in j.parameters['vars'])
+ self.assertEqual(rp, set(['org/project', 'org/project0',
+ 'org/project0']))
+ j = self.getJobFromHistory('child1')
+ rp = set([p['name'] for p in j.parameters['projects']])
+ self.assertEqual(j.parameters['vars']['override'], 1)
+ self.assertEqual(j.parameters['vars']['child1override'], 1)
+ self.assertEqual(j.parameters['vars']['parent'], 0)
+ self.assertEqual(j.parameters['vars']['child1'], 1)
+ self.assertFalse('child2' in j.parameters['vars'])
+ self.assertFalse('child3' in j.parameters['vars'])
+ self.assertEqual(rp, set(['org/project', 'org/project0',
+ 'org/project1']))
+ j = self.getJobFromHistory('child2')
+ rp = set([p['name'] for p in j.parameters['projects']])
+ self.assertEqual(j.parameters['vars']['override'], 2)
+ self.assertEqual(j.parameters['vars']['child1override'], 0)
+ self.assertEqual(j.parameters['vars']['parent'], 0)
+ self.assertFalse('child1' in j.parameters['vars'])
+ self.assertEqual(j.parameters['vars']['child2'], 2)
+ self.assertFalse('child3' in j.parameters['vars'])
+ self.assertEqual(rp, set(['org/project', 'org/project0',
+ 'org/project2']))
+ j = self.getJobFromHistory('child3')
+ rp = set([p['name'] for p in j.parameters['projects']])
+ self.assertEqual(j.parameters['vars']['override'], 3)
+ self.assertEqual(j.parameters['vars']['child1override'], 0)
+ self.assertEqual(j.parameters['vars']['parent'], 0)
+ self.assertFalse('child1' in j.parameters['vars'])
+ self.assertFalse('child2' in j.parameters['vars'])
+ self.assertEqual(j.parameters['vars']['child3'], 3)
+ self.assertEqual(rp, set(['org/project', 'org/project0',
+ 'org/project3']))
+
def test_queue_names(self):
"Test shared change queue names"
tenant = self.sched.abide.tenants.get('tenant-one')
diff --git a/zuul/configloader.py b/zuul/configloader.py
index c1a65be..6a9ba01 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -383,57 +383,55 @@
class JobParser(object):
ANSIBLE_ROLE_RE = re.compile(r'^(ansible[-_.+]*)*(role[-_.+]*)*')
- @staticmethod
- def getSchema():
- zuul_role = {vs.Required('zuul'): str,
- 'name': str}
+ zuul_role = {vs.Required('zuul'): str,
+ 'name': str}
- galaxy_role = {vs.Required('galaxy'): str,
- 'name': str}
+ galaxy_role = {vs.Required('galaxy'): str,
+ 'name': str}
- role = vs.Any(zuul_role, galaxy_role)
+ role = vs.Any(zuul_role, galaxy_role)
- job_project = {vs.Required('name'): str,
- 'override-branch': str}
+ job_project = {vs.Required('name'): str,
+ 'override-branch': str}
- secret = {vs.Required('name'): str,
- vs.Required('secret'): str}
+ secret = {vs.Required('name'): str,
+ vs.Required('secret'): str}
- job = {vs.Required('name'): str,
- 'parent': vs.Any(str, None),
- 'final': bool,
- 'failure-message': str,
- 'success-message': str,
- 'failure-url': str,
- 'success-url': str,
- 'hold-following-changes': bool,
- 'voting': bool,
- 'semaphore': str,
- 'tags': to_list(str),
- 'branches': to_list(str),
- 'files': to_list(str),
- 'secrets': to_list(vs.Any(secret, str)),
- 'irrelevant-files': to_list(str),
- # validation happens in NodeSetParser
- 'nodeset': vs.Any(dict, str),
- 'timeout': int,
- 'attempts': int,
- 'pre-run': to_list(str),
- 'post-run': to_list(str),
- 'run': str,
- '_source_context': model.SourceContext,
- '_start_mark': ZuulMark,
- 'roles': to_list(role),
- 'required-projects': to_list(vs.Any(job_project, str)),
- 'vars': dict,
- 'dependencies': to_list(str),
- 'allowed-projects': to_list(str),
- 'override-branch': str,
- 'description': str,
- 'post-review': bool
- }
+ job = {vs.Required('name'): str,
+ 'parent': vs.Any(str, None),
+ 'final': bool,
+ 'failure-message': str,
+ 'success-message': str,
+ 'failure-url': str,
+ 'success-url': str,
+ 'hold-following-changes': bool,
+ 'voting': bool,
+ 'semaphore': str,
+ 'tags': to_list(str),
+ 'branches': to_list(str),
+ 'files': to_list(str),
+ 'secrets': to_list(vs.Any(secret, str)),
+ 'irrelevant-files': to_list(str),
+ # validation happens in NodeSetParser
+ 'nodeset': vs.Any(dict, str),
+ 'timeout': int,
+ 'attempts': int,
+ 'pre-run': to_list(str),
+ 'post-run': to_list(str),
+ 'run': str,
+ '_source_context': model.SourceContext,
+ '_start_mark': ZuulMark,
+ 'roles': to_list(role),
+ 'required-projects': to_list(vs.Any(job_project, str)),
+ 'vars': dict,
+ 'dependencies': to_list(str),
+ 'allowed-projects': to_list(str),
+ 'override-branch': str,
+ 'description': str,
+ 'post-review': bool
+ }
- return vs.Schema(job)
+ schema = vs.Schema(job)
simple_attributes = [
'final',
@@ -478,7 +476,7 @@
@staticmethod
def fromYaml(tenant, layout, conf, project_pipeline=False):
with configuration_exceptions('job', conf):
- JobParser.getSchema()(conf)
+ JobParser.schema(conf)
# NB: The default detection system in the Job class requires
# that we always assign values directly rather than modifying
@@ -710,10 +708,13 @@
class ProjectTemplateParser(object):
- log = logging.getLogger("zuul.ProjectTemplateParser")
+ def __init__(self, tenant, layout):
+ self.log = logging.getLogger("zuul.ProjectTemplateParser")
+ self.tenant = tenant
+ self.layout = layout
+ self.schema = self.getSchema()
- @staticmethod
- def getSchema(layout):
+ def getSchema(self):
project_template = {
vs.Required('name'): str,
'description': str,
@@ -724,40 +725,31 @@
'_start_mark': ZuulMark,
}
- for p in layout.pipelines.values():
+ for p in self.layout.pipelines.values():
project_template[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
return vs.Schema(project_template)
- @staticmethod
- def fromYaml(tenant, layout, conf, template):
- if template:
- project_or_template = 'project-template'
- else:
- project_or_template = 'project'
- with configuration_exceptions(project_or_template, conf):
- ProjectTemplateParser.getSchema(layout)(conf)
- # Make a copy since we modify this later via pop
- conf = copy.deepcopy(conf)
+ def fromYaml(self, conf, validate=True):
+ if validate:
+ with configuration_exceptions('project-template', conf):
+ self.schema(conf)
project_template = model.ProjectConfig(conf['name'])
source_context = conf['_source_context']
start_mark = conf['_start_mark']
- for pipeline in layout.pipelines.values():
+ for pipeline in self.layout.pipelines.values():
conf_pipeline = conf.get(pipeline.name)
if not conf_pipeline:
continue
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
- ProjectTemplateParser._parseJobList(
- tenant, layout, conf_pipeline.get('jobs', []),
- source_context, start_mark, project_pipeline.job_list,
- template)
+ self.parseJobList(
+ conf_pipeline.get('jobs', []),
+ source_context, start_mark, project_pipeline.job_list)
return project_template
- @staticmethod
- def _parseJobList(tenant, layout, conf, source_context,
- start_mark, job_list, template):
+ def parseJobList(self, conf, source_context, start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, str):
attrs = dict(name=conf_job)
@@ -778,17 +770,21 @@
# validate that the job is existing
with configuration_exceptions('project or project-template',
attrs):
- layout.getJob(attrs['name'])
+ self.layout.getJob(attrs['name'])
- job_list.addJob(JobParser.fromYaml(tenant, layout, attrs,
- project_pipeline=True))
+ job_list.addJob(JobParser.fromYaml(self.tenant, self.layout,
+ attrs, project_pipeline=True))
class ProjectParser(object):
- log = logging.getLogger("zuul.ProjectParser")
+ def __init__(self, tenant, layout, project_template_parser):
+ self.log = logging.getLogger("zuul.ProjectParser")
+ self.tenant = tenant
+ self.layout = layout
+ self.project_template_parser = project_template_parser
+ self.schema = self.getSchema()
- @staticmethod
- def getSchema(layout):
+ def getSchema(self):
project = {
vs.Required('name'): str,
'description': str,
@@ -800,20 +796,19 @@
'_start_mark': ZuulMark,
}
- for p in layout.pipelines.values():
+ for p in self.layout.pipelines.values():
project[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
return vs.Schema(project)
- @staticmethod
- def fromYaml(tenant, layout, conf_list):
+ def fromYaml(self, conf_list):
for conf in conf_list:
with configuration_exceptions('project', conf):
- ProjectParser.getSchema(layout)(conf)
+ self.schema(conf)
with configuration_exceptions('project', conf_list[0]):
project_name = conf_list[0]['name']
- (trusted, project) = tenant.getProject(project_name)
+ (trusted, project) = self.tenant.getProject(project_name)
if project is None:
raise ProjectNotFoundError(project_name)
project_config = model.ProjectConfig(project.canonical_name)
@@ -826,23 +821,21 @@
if project != conf['_source_context'].project:
raise ProjectNotPermittedError()
- # Make a copy since we modify this later via pop
- conf = copy.deepcopy(conf)
- conf_templates = conf.pop('templates', [])
+ conf_templates = conf.get('templates', [])
# The way we construct a project definition is by
# parsing the definition as a template, then applying
# all of the templates, including the newly parsed
# one, in order.
- project_template = ProjectTemplateParser.fromYaml(
- tenant, layout, conf, template=False)
+ project_template = self.project_template_parser.fromYaml(
+ conf, validate=False)
# If this project definition is in a place where it
# should get implied branch matchers, set it.
if (not conf['_source_context'].trusted):
implied_branch = conf['_source_context'].branch
for name in conf_templates:
- if name not in layout.project_templates:
+ if name not in self.layout.project_templates:
raise TemplateNotFoundError(name)
- configs.extend([(layout.project_templates[name],
+ configs.extend([(self.layout.project_templates[name],
implied_branch)
for name in conf_templates])
configs.append((project_template, implied_branch))
@@ -860,7 +853,7 @@
project_config.merge_mode = model.MERGER_MAP['merge-resolve']
if project_config.default_branch is None:
project_config.default_branch = 'master'
- for pipeline in layout.pipelines.values():
+ for pipeline in self.layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
queue_name = None
# For every template, iterate over the job tree and replace or
@@ -1527,13 +1520,15 @@
continue
layout.addSemaphore(semaphore)
+ project_template_parser = ProjectTemplateParser(tenant, layout)
for config_template in data.project_templates:
classes = TenantParser._getLoadClasses(tenant, config_template)
if 'project-template' not in classes:
continue
- layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
- tenant, layout, config_template, template=True))
+ layout.addProjectTemplate(project_template_parser.fromYaml(
+ config_template))
+ project_parser = ProjectParser(tenant, layout, project_template_parser)
for config_projects in data.projects.values():
# Unlike other config classes, we expect multiple project
# stanzas with the same name, so that a config repo can
@@ -1551,8 +1546,8 @@
if not filtered_projects:
continue
- layout.addProjectConfig(ProjectParser.fromYaml(
- tenant, layout, filtered_projects))
+ layout.addProjectConfig(project_parser.fromYaml(
+ filtered_projects))
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):
diff --git a/zuul/model.py b/zuul/model.py
index c5d0a4d..c95a169 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -877,7 +877,7 @@
def __getattr__(self, name):
v = self.__dict__.get(name)
if v is None:
- return copy.deepcopy(self.attributes[name])
+ return self.attributes[name]
return v
def _get(self, name):
@@ -925,13 +925,13 @@
self.branch_matcher = change_matcher.MatchAny(matchers)
def updateVariables(self, other_vars):
- v = self.variables
+ v = copy.deepcopy(self.variables)
Job._deepUpdate(v, other_vars)
self.variables = v
def updateProjects(self, other_projects):
- required_projects = self.required_projects
- Job._deepUpdate(required_projects, other_projects)
+ required_projects = self.required_projects.copy()
+ required_projects.update(other_projects)
self.required_projects = required_projects
@staticmethod
@@ -958,7 +958,7 @@
# copy all attributes
for k in self.inheritable_attributes:
if (other._get(k) is not None):
- setattr(self, k, copy.deepcopy(getattr(other, k)))
+ setattr(self, k, getattr(other, k))
msg = 'inherit from %s' % (repr(other),)
self.inheritance_path = other.inheritance_path + (msg,)