Add an additional pass through project templates
In order to find project templates that need to be expanded from job
matchers that are not project specific, we need to make a pass through
all of the projects assuming all templates need to be expanded. We then
look at the result of the expansion to see if anything was actually
done. As part of this, we also collect same-expansions on a job basis to
track if a given job always has the same matcher expansion. If it does,
we can apply that to the job definition and not to the project-pipeline
defintion, which could lower the number of templates that need to be
expanded.
This may be the ugliest code I've ever written. I'm sorry.
Also fix a bash bug in the run-migration script that caused final to
always get run regardless of flag setting. Whoops.
Change-Id: I523909e5242e0db125b7560cbdcd9ac41ca6c72f
diff --git a/tools/run-migration.sh b/tools/run-migration.sh
index be297f4..618fc56 100755
--- a/tools/run-migration.sh
+++ b/tools/run-migration.sh
@@ -47,12 +47,12 @@
BASE_DIR=$(cd $(dirname $0)/../..; pwd)
cd $BASE_DIR/project-config
-if [[ $FINAL ]] ; then
+if [[ $FINAL = 1 ]] ; then
git reset --hard
fi
python3 $BASE_DIR/zuul/zuul/cmd/migrate.py --mapping=zuul/mapping.yaml \
zuul/layout.yaml jenkins/jobs nodepool/nodepool.yaml . $VERBOSE
-if [[ $FINAL ]] ; then
+if [[ $FINAL = 1 ]] ; then
find ../openstack-zuul-jobs/playbooks/legacy -maxdepth 1 -mindepth 1 \
-type d | xargs rm -rf
mv zuul.d/zuul-legacy-* ../openstack-zuul-jobs/zuul.d/
diff --git a/zuul/cmd/migrate.py b/zuul/cmd/migrate.py
index 3ab3902..d3745a0 100644
--- a/zuul/cmd/migrate.py
+++ b/zuul/cmd/migrate.py
@@ -42,6 +42,9 @@
import jenkins_jobs.parser
import yaml
+JOB_MATCHERS = {} # type: Dict[str, Dict[str, Dict]]
+TEMPLATES_TO_EXPAND = {} # type: Dict[str, List]
+JOBS_FOR_EXPAND = collections.defaultdict(dict) # type: ignore
JOBS_BY_ORIG_TEMPLATE = {} # type: ignore
SUFFIXES = [] # type: ignore
ENVIRONMENT = '{{ zuul | zuul_legacy_vars }}'
@@ -186,6 +189,37 @@
return
+def normalize_project_expansions():
+ remove_from_job_matchers = []
+ template = None
+ # First find the matchers that are the same for all jobs
+ for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
+ JOB_MATCHERS[job_name] = None
+ for project_name, expansion in project.items():
+ template = expansion['template']
+ if not JOB_MATCHERS[job_name]:
+ JOB_MATCHERS[job_name] = copy.deepcopy(expansion['info'])
+ else:
+ if JOB_MATCHERS[job_name] != expansion['info']:
+ # We have different expansions for this job, it can't be
+ # done at the job level
+ remove_from_job_matchers.append(job_name)
+
+ for job_name in remove_from_job_matchers:
+ JOB_MATCHERS.pop(job_name, None)
+
+ # Second, find out which projects need to expand a given template
+ for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
+ # There is a job-level expansion for this one
+ if job_name in JOB_MATCHERS.keys():
+ continue
+ for project_name, expansion in project.items():
+ TEMPLATES_TO_EXPAND[project_name] = []
+ if expansion['info']:
+ # There is an expansion for this project
+ TEMPLATES_TO_EXPAND[project_name].append(expansion['template'])
+
+
# from :
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data flake8: noqa
def should_use_block(value):
@@ -910,6 +944,14 @@
if expanded_projects:
output['required-projects'] = sorted(list(set(expanded_projects)))
+ if self.name in JOB_MATCHERS:
+ for k, v in JOB_MATCHERS[self.name].items():
+ if k in output:
+ self.log.error(
+ 'Job %s has attributes directly and from matchers',
+ self.name)
+ output[k] = v
+
return output
def toPipelineDict(self):
@@ -1345,7 +1387,7 @@
for pipeline, value in template.items():
if pipeline == 'name':
continue
- if pipeline not in project:
+ if pipeline not in project or 'jobs' not in project[pipeline]:
project[pipeline] = dict(jobs=[])
project[pipeline]['jobs'].extend(value['jobs'])
@@ -1355,7 +1397,7 @@
return job.orig
return None
- def applyProjectMatchers(self, matchers, project):
+ def applyProjectMatchers(self, matchers, project, final=False):
'''
Apply per-project job matchers to the given project.
@@ -1373,7 +1415,8 @@
self.log.debug(
"Applied irrelevant-files to job %s in project %s",
job, project['name'])
- job = {job: {'irrelevant-files': list(set(files))}}
+ job = {job: {'irrelevant-files':
+ sorted(list(set(files)))}}
elif isinstance(job, dict):
job = job.copy()
job_name = get_single_key(job)
@@ -1387,8 +1430,8 @@
if 'irrelevant-files' not in extras:
extras['irrelevant-files'] = []
extras['irrelevant-files'].extend(files)
- extras['irrelevant-files'] = list(
- set(extras['irrelevant-files']))
+ extras['irrelevant-files'] = sorted(list(
+ set(extras['irrelevant-files'])))
job[job_name] = extras
new_jobs.append(job)
return new_jobs
@@ -1398,17 +1441,61 @@
if k in ('templates', 'name'):
continue
project[k]['jobs'] = processPipeline(
- project[k]['jobs'], job_name_regex, files)
+ project[k].get('jobs', []), job_name_regex, files)
- for matcher in matchers:
- # find the project-specific section
- for skipper in matcher.get('skip-if', []):
- if skipper.get('project'):
- if re.search(skipper['project'], project['name']):
- if 'all-files-match-any' in skipper:
- applyIrrelevantFiles(
- matcher['name'],
- skipper['all-files-match-any'])
+ if matchers:
+ for matcher in matchers:
+ # find the project-specific section
+ for skipper in matcher.get('skip-if', []):
+ if skipper.get('project'):
+ if re.search(skipper['project'], project['name']):
+ if 'all-files-match-any' in skipper:
+ applyIrrelevantFiles(
+ matcher['name'],
+ skipper['all-files-match-any'])
+
+ if not final:
+ return
+
+ for k, v in project.items():
+ if k in ('templates', 'name'):
+ continue
+ jobs = []
+ for job in project[k].get('jobs', []):
+ if isinstance(job, dict):
+ job_name = get_single_key(job)
+ else:
+ job_name = job
+ if job_name in JOB_MATCHERS:
+ jobs.append(job)
+ continue
+ orig_name = self.getOldJobName(job_name)
+ if not orig_name:
+ jobs.append(job)
+ continue
+ orig_name = orig_name.format(
+ name=project['name'].split('/')[1])
+ info = {}
+ for layout_job in self.mapping.layout.get('jobs', []):
+ if 'parameter-function' in layout_job:
+ continue
+ if 'skip-if' in layout_job:
+ continue
+ if re.search(layout_job['name'], orig_name):
+ if not layout_job.get('voting', True):
+ info['voting'] = False
+ if layout_job.get('branch'):
+ info['branches'] = layout_job['branch']
+ if layout_job.get('files'):
+ info['files'] = layout_job['files']
+ if not isinstance(job, dict):
+ job = {job: info}
+ else:
+ job[job_name].update(info)
+
+ jobs.append(job)
+ if jobs:
+ project[k]['jobs'] = jobs
def writeProject(self, project):
'''
@@ -1423,12 +1510,7 @@
if 'name' in project:
new_project['name'] = project['name']
- job_matchers = self.scanForProjectMatchers(project['name'])
- if job_matchers:
- exp_template_names = self.findReferencedTemplateNames(
- job_matchers, project['name'])
- else:
- exp_template_names = []
+ exp_template_names = TEMPLATES_TO_EXPAND.get(project['name'], [])
templates_to_expand = []
if 'template' in project:
@@ -1447,6 +1529,51 @@
new_project[key] = collections.OrderedDict()
if key == 'gate':
for queue in self.change_queues:
+ if (project['name'] not in queue.getProjects() or
+ len(queue.getProjects()) == 1):
+ continue
+ new_project[key]['queue'] = queue.name
+ tmp = [job for job in self.makeNewJobs(value)]
+ # Don't insert into self.job_objects - that was done
+ # in the speculative pass
+ jobs = [job.toPipelineDict() for job in tmp]
+ if jobs:
+ new_project[key]['jobs'] = jobs
+ if not new_project[key]:
+ del new_project[key]
+
+ for name in templates_to_expand:
+ self.expandTemplateIntoProject(name, new_project)
+
+ job_matchers = self.scanForProjectMatchers(project['name'])
+
+ # Need a deep copy after expansion, else our templates end up
+ # also getting this change.
+ new_project = copy.deepcopy(new_project)
+ self.applyProjectMatchers(job_matchers, new_project, final=True)
+
+ return new_project
+
+ def checkSpeculativeProject(self, project):
+ '''
+ Create a new v3 project definition expanding all templates.
+ '''
+ new_project = collections.OrderedDict()
+ if 'name' in project:
+ new_project['name'] = project['name']
+
+ templates_to_expand = []
+ for template in project.get('template', []):
+ templates_to_expand.append(template['name'])
+
+ # We have to do this section to expand self.job_objects
+ for key, value in project.items():
+ if key in ('name', 'template'):
+ continue
+ else:
+ new_project[key] = collections.OrderedDict()
+ if key == 'gate':
+ for queue in self.change_queues:
if project['name'] not in queue.getProjects():
continue
if len(queue.getProjects()) == 1:
@@ -1454,18 +1581,60 @@
new_project[key]['queue'] = queue.name
tmp = [job for job in self.makeNewJobs(value)]
self.job_objects.extend(tmp)
- jobs = [job.toPipelineDict() for job in tmp]
- new_project[key]['jobs'] = jobs
for name in templates_to_expand:
- self.expandTemplateIntoProject(name, new_project)
- # Need a deep copy after expansion, else our templates end up
- # also getting this change.
- new_project = copy.deepcopy(new_project)
- self.applyProjectMatchers(job_matchers, new_project)
+ expand_project = copy.deepcopy(new_project)
+ self.expandTemplateIntoProject(name, expand_project)
- return new_project
+ # Need a deep copy after expansion, else our templates end up
+ # also getting this change.
+ expand_project = copy.deepcopy(expand_project)
+ job_matchers = self.scanForProjectMatchers(project['name'])
+ self.applyProjectMatchers(job_matchers, expand_project)
+
+ # We should now have a project-pipeline with only the
+ # jobs expanded from this one template
+ for project_part in expand_project.values():
+ # The pipelines are dicts - we only want pipelines
+ if isinstance(project_part, dict):
+ if 'jobs' not in project_part:
+ continue
+ self.processProjectTemplateExpansion(
+ project_part, project, name)
+
+ def processProjectTemplateExpansion(self, project_part, project, template):
+ # project_part should be {'jobs': []}
+ job_list = project_part['jobs']
+ for new_job in job_list:
+ if isinstance(new_job, dict):
+ new_job_name = get_single_key(new_job)
+ info = new_job[new_job_name]
+ else:
+ new_job_name = new_job
+ info = None
+ orig_name = self.getOldJobName(new_job_name)
+ if not orig_name:
+ self.log.error("Job without old name: %s", new_job_name)
+ continue
+ orig_name = orig_name.format(name=project['name'].split('/')[1])
+
+ for layout_job in self.mapping.layout.get('jobs', []):
+ if 'parameter-function' in layout_job:
+ continue
+ if re.search(layout_job['name'], orig_name):
+ if not info:
+ info = {}
+ if not layout_job.get('voting', True):
+ info['voting'] = False
+ if layout_job.get('branch'):
+ info['branches'] = layout_job['branch']
+ if layout_job.get('files'):
+ info['files'] = layout_job['files']
+
+ if info:
+ expansion = dict(info=info, template=template)
+ JOBS_FOR_EXPAND[new_job_name][project['name']] = expansion
def writeJobs(self):
output_dir = self.setupDir()
@@ -1487,13 +1656,17 @@
template_config,
key=lambda template: template['project-template']['name'])
+ for project in self.layout.get('projects', []):
+ self.checkSpeculativeProject(project)
+ normalize_project_expansions()
+
project_names = []
for project in self.layout.get('projects', []):
project_names.append(project['name'])
project_dict = self.writeProject(project)
merge_project_dict(
project_dicts, project['name'],
- self.writeProject(project))
+ project_dict)
project_config = project_dicts_to_list(project_dicts)
seen_jobs = []