Merge "Add an additional pass through project templates" into feature/zuulv3
diff --git a/tools/run-migration.sh b/tools/run-migration.sh
index be297f4..618fc56 100755
--- a/tools/run-migration.sh
+++ b/tools/run-migration.sh
@@ -47,12 +47,12 @@
BASE_DIR=$(cd $(dirname $0)/../..; pwd)
cd $BASE_DIR/project-config
-if [[ $FINAL ]] ; then
+if [[ $FINAL = 1 ]] ; then
git reset --hard
fi
python3 $BASE_DIR/zuul/zuul/cmd/migrate.py --mapping=zuul/mapping.yaml \
zuul/layout.yaml jenkins/jobs nodepool/nodepool.yaml . $VERBOSE
-if [[ $FINAL ]] ; then
+if [[ $FINAL = 1 ]] ; then
find ../openstack-zuul-jobs/playbooks/legacy -maxdepth 1 -mindepth 1 \
-type d | xargs rm -rf
mv zuul.d/zuul-legacy-* ../openstack-zuul-jobs/zuul.d/
diff --git a/zuul/cmd/migrate.py b/zuul/cmd/migrate.py
index 1f0a602..d3745a0 100644
--- a/zuul/cmd/migrate.py
+++ b/zuul/cmd/migrate.py
@@ -42,6 +42,9 @@
import jenkins_jobs.parser
import yaml
+JOB_MATCHERS = {} # type: Dict[str, Dict[str, Dict]]
+TEMPLATES_TO_EXPAND = {} # type: Dict[str, List]
+JOBS_FOR_EXPAND = collections.defaultdict(dict) # type: ignore
JOBS_BY_ORIG_TEMPLATE = {} # type: ignore
SUFFIXES = [] # type: ignore
ENVIRONMENT = '{{ zuul | zuul_legacy_vars }}'
@@ -186,6 +189,37 @@
return
+def normalize_project_expansions():
+ remove_from_job_matchers = []
+ template = None
+ # First find the matchers that are the same for all jobs
+ for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
+ JOB_MATCHERS[job_name] = None
+ for project_name, expansion in project.items():
+ template = expansion['template']
+ if not JOB_MATCHERS[job_name]:
+ JOB_MATCHERS[job_name] = copy.deepcopy(expansion['info'])
+ else:
+ if JOB_MATCHERS[job_name] != expansion['info']:
+ # We have different expansions for this job, it can't be
+ # done at the job level
+ remove_from_job_matchers.append(job_name)
+
+ for job_name in remove_from_job_matchers:
+ JOB_MATCHERS.pop(job_name, None)
+
+ # Second, find out which projects need to expand a given template
+ for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
+ # There is a job-level expansion for this one
+ if job_name in JOB_MATCHERS.keys():
+ continue
+ for project_name, expansion in project.items():
+ TEMPLATES_TO_EXPAND[project_name] = []
+ if expansion['info']:
+ # There is an expansion for this project
+ TEMPLATES_TO_EXPAND[project_name].append(expansion['template'])
+
+
# from :
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data flake8: noqa
def should_use_block(value):
@@ -910,6 +944,14 @@
if expanded_projects:
output['required-projects'] = sorted(list(set(expanded_projects)))
+ if self.name in JOB_MATCHERS:
+ for k, v in JOB_MATCHERS[self.name].items():
+ if k in output:
+ self.log.error(
+ 'Job %s has attributes directly and from matchers',
+ self.name)
+ output[k] = v
+
return output
def toPipelineDict(self):
@@ -1345,7 +1387,7 @@
for pipeline, value in template.items():
if pipeline == 'name':
continue
- if pipeline not in project:
+ if pipeline not in project or 'jobs' not in project[pipeline]:
project[pipeline] = dict(jobs=[])
project[pipeline]['jobs'].extend(value['jobs'])
@@ -1355,7 +1397,7 @@
return job.orig
return None
- def applyProjectMatchers(self, matchers, project):
+ def applyProjectMatchers(self, matchers, project, final=False):
'''
Apply per-project job matchers to the given project.
@@ -1373,7 +1415,8 @@
self.log.debug(
"Applied irrelevant-files to job %s in project %s",
job, project['name'])
- job = {job: {'irrelevant-files': list(set(files))}}
+ job = {job: {'irrelevant-files':
+ sorted(list(set(files)))}}
elif isinstance(job, dict):
job = job.copy()
job_name = get_single_key(job)
@@ -1387,8 +1430,8 @@
if 'irrelevant-files' not in extras:
extras['irrelevant-files'] = []
extras['irrelevant-files'].extend(files)
- extras['irrelevant-files'] = list(
- set(extras['irrelevant-files']))
+ extras['irrelevant-files'] = sorted(list(
+ set(extras['irrelevant-files'])))
job[job_name] = extras
new_jobs.append(job)
return new_jobs
@@ -1398,17 +1441,61 @@
if k in ('templates', 'name'):
continue
project[k]['jobs'] = processPipeline(
- project[k]['jobs'], job_name_regex, files)
+ project[k].get('jobs', []), job_name_regex, files)
- for matcher in matchers:
- # find the project-specific section
- for skipper in matcher.get('skip-if', []):
- if skipper.get('project'):
- if re.search(skipper['project'], project['name']):
- if 'all-files-match-any' in skipper:
- applyIrrelevantFiles(
- matcher['name'],
- skipper['all-files-match-any'])
+ if matchers:
+ for matcher in matchers:
+ # find the project-specific section
+ for skipper in matcher.get('skip-if', []):
+ if skipper.get('project'):
+ if re.search(skipper['project'], project['name']):
+ if 'all-files-match-any' in skipper:
+ applyIrrelevantFiles(
+ matcher['name'],
+ skipper['all-files-match-any'])
+
+ if not final:
+ return
+
+ for k, v in project.items():
+ if k in ('templates', 'name'):
+ continue
+ jobs = []
+ for job in project[k].get('jobs', []):
+ if isinstance(job, dict):
+ job_name = get_single_key(job)
+ else:
+ job_name = job
+ if job_name in JOB_MATCHERS:
+ jobs.append(job)
+ continue
+ orig_name = self.getOldJobName(job_name)
+ if not orig_name:
+ jobs.append(job)
+ continue
+ orig_name = orig_name.format(
+ name=project['name'].split('/')[1])
+ info = {}
+ for layout_job in self.mapping.layout.get('jobs', []):
+ if 'parameter-function' in layout_job:
+ continue
+ if 'skip-if' in layout_job:
+ continue
+ if re.search(layout_job['name'], orig_name):
+ if not layout_job.get('voting', True):
+ info['voting'] = False
+ if layout_job.get('branch'):
+ info['branches'] = layout_job['branch']
+ if layout_job.get('files'):
+ info['files'] = layout_job['files']
+ if not isinstance(job, dict):
+ job = {job: info}
+ else:
+ job[job_name].update(info)
+
+ jobs.append(job)
+ if jobs:
+ project[k]['jobs'] = jobs
def writeProject(self, project):
'''
@@ -1423,12 +1510,7 @@
if 'name' in project:
new_project['name'] = project['name']
- job_matchers = self.scanForProjectMatchers(project['name'])
- if job_matchers:
- exp_template_names = self.findReferencedTemplateNames(
- job_matchers, project['name'])
- else:
- exp_template_names = []
+ exp_template_names = TEMPLATES_TO_EXPAND.get(project['name'], [])
templates_to_expand = []
if 'template' in project:
@@ -1447,6 +1529,51 @@
new_project[key] = collections.OrderedDict()
if key == 'gate':
for queue in self.change_queues:
+ if (project['name'] not in queue.getProjects() or
+ len(queue.getProjects()) == 1):
+ continue
+ new_project[key]['queue'] = queue.name
+ tmp = [job for job in self.makeNewJobs(value)]
+ # Don't insert into self.job_objects - that was done
+ # in the speculative pass
+ jobs = [job.toPipelineDict() for job in tmp]
+ if jobs:
+ new_project[key]['jobs'] = jobs
+ if not new_project[key]:
+ del new_project[key]
+
+ for name in templates_to_expand:
+ self.expandTemplateIntoProject(name, new_project)
+
+ job_matchers = self.scanForProjectMatchers(project['name'])
+
+ # Need a deep copy after expansion, else our templates end up
+ # also getting this change.
+ new_project = copy.deepcopy(new_project)
+ self.applyProjectMatchers(job_matchers, new_project, final=True)
+
+ return new_project
+
+ def checkSpeculativeProject(self, project):
+ '''
+ Create a new v3 project definition expanding all templates.
+ '''
+ new_project = collections.OrderedDict()
+ if 'name' in project:
+ new_project['name'] = project['name']
+
+ templates_to_expand = []
+ for template in project.get('template', []):
+ templates_to_expand.append(template['name'])
+
+ # We have to do this section to expand self.job_objects
+ for key, value in project.items():
+ if key in ('name', 'template'):
+ continue
+ else:
+ new_project[key] = collections.OrderedDict()
+ if key == 'gate':
+ for queue in self.change_queues:
if project['name'] not in queue.getProjects():
continue
if len(queue.getProjects()) == 1:
@@ -1454,18 +1581,60 @@
new_project[key]['queue'] = queue.name
tmp = [job for job in self.makeNewJobs(value)]
self.job_objects.extend(tmp)
- jobs = [job.toPipelineDict() for job in tmp]
- new_project[key]['jobs'] = jobs
for name in templates_to_expand:
- self.expandTemplateIntoProject(name, new_project)
- # Need a deep copy after expansion, else our templates end up
- # also getting this change.
- new_project = copy.deepcopy(new_project)
- self.applyProjectMatchers(job_matchers, new_project)
+ expand_project = copy.deepcopy(new_project)
+ self.expandTemplateIntoProject(name, expand_project)
- return new_project
+ # Need a deep copy after expansion, else our templates end up
+ # also getting this change.
+ expand_project = copy.deepcopy(expand_project)
+ job_matchers = self.scanForProjectMatchers(project['name'])
+ self.applyProjectMatchers(job_matchers, expand_project)
+
+ # We should now have a project-pipeline with only the
+ # jobs expanded from this one template
+ for project_part in expand_project.values():
+ # The pipelines are dicts - we only want pipelines
+ if isinstance(project_part, dict):
+ if 'jobs' not in project_part:
+ continue
+ self.processProjectTemplateExpansion(
+ project_part, project, name)
+
+ def processProjectTemplateExpansion(self, project_part, project, template):
+ # project_part should be {'jobs': []}
+ job_list = project_part['jobs']
+ for new_job in job_list:
+ if isinstance(new_job, dict):
+ new_job_name = get_single_key(new_job)
+ info = new_job[new_job_name]
+ else:
+ new_job_name = new_job
+ info = None
+ orig_name = self.getOldJobName(new_job_name)
+ if not orig_name:
+ self.log.error("Job without old name: %s", new_job_name)
+ continue
+ orig_name = orig_name.format(name=project['name'].split('/')[1])
+
+ for layout_job in self.mapping.layout.get('jobs', []):
+ if 'parameter-function' in layout_job:
+ continue
+ if re.search(layout_job['name'], orig_name):
+ if not info:
+ info = {}
+ if not layout_job.get('voting', True):
+ info['voting'] = False
+ if layout_job.get('branch'):
+ info['branches'] = layout_job['branch']
+ if layout_job.get('files'):
+ info['files'] = layout_job['files']
+
+ if info:
+ expansion = dict(info=info, template=template)
+ JOBS_FOR_EXPAND[new_job_name][project['name']] = expansion
def writeJobs(self):
output_dir = self.setupDir()
@@ -1487,6 +1656,10 @@
template_config,
key=lambda template: template['project-template']['name'])
+ for project in self.layout.get('projects', []):
+ self.checkSpeculativeProject(project)
+ normalize_project_expansions()
+
project_names = []
for project in self.layout.get('projects', []):
project_names.append(project['name'])