Merge "Remove unused function validate_conf" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index 041681a..a87c196 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -36,6 +36,8 @@
irrelevant-files:
- zuul/cmd/migrate.py
- playbooks/zuul-migrate/.*
+ vars:
+ sphinx_python: python3
- tox-cover:
irrelevant-files:
- zuul/cmd/migrate.py
@@ -53,6 +55,8 @@
irrelevant-files:
- zuul/cmd/migrate.py
- playbooks/zuul-migrate/.*
+ vars:
+ sphinx_python: python3
- tox-pep8
- tox-py35:
irrelevant-files:
@@ -61,5 +65,7 @@
- zuul-stream-functional
post:
jobs:
- - publish-openstack-sphinx-docs-infra
+ - publish-openstack-sphinx-docs-infra:
+ vars:
+ sphinx_python: python3
- publish-openstack-python-branch-tarball
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 173e615..916e66a 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -1097,6 +1097,14 @@
changes which break the others. This is a free-form string;
just set the same value for each group of projects.
+ .. attr:: debug
+
+ If this is set to `true`, Zuul will include debugging
+ information in reports it makes about items in the pipeline.
+ This should not normally be set, but in situations were it is
+ difficult to determine why Zuul did or did not run a certain
+ job, the additional information this provides may help.
+
.. _project-template:
Project Template
@@ -1318,7 +1326,7 @@
.. attr:: pragma
- The pragma item currently only supports one attribute:
+ The pragma item currently supports the following attributes:
.. attr:: implied-branch-matchers
@@ -1333,3 +1341,43 @@
Note that if a job contains an explicit branch matcher, it will
be used regardless of the value supplied here.
+
+ .. attr:: implied-branches
+
+ This is a list of regular expressions, just as
+ :attr:`job.branches`, which may be used to supply the value of
+ the implied branch matcher for all jobs in a file.
+
+ This may be useful if two projects share jobs but have
+ dissimilar branch names. If, for example, two projects have
+ stable maintenance branches with dissimilar names, but both
+ should use the same job variants, this directive may be used to
+ indicate that all of the jobs defined in the stable branch of
+ the first project may also be used for the stable branch of the
+ other. For example:
+
+ .. code-block:: yaml
+
+ - pragma:
+ implied-branches:
+ - stable/foo
+ - stable/bar
+
+ The above code, when added to the ``stable/foo`` branch of a
+ project would indicate that the job variants described in that
+ file should not only be used for changes to ``stable/foo``, but
+ also on changes to ``stable/bar``, which may be in another
+ project.
+
+ Note that if a job contains an explicit branch matcher, it will
+ be used regardless of the value supplied here.
+
+ Note also that the presence of `implied-branches` does not
+ automatically set `implied-branch-matchers`. Zuul will still
+ decide if implied branch matchers are warranted at all, using
+ the heuristics described in :attr:`job.branches`, and only use
+ the value supplied here if that is the case. If you want to
+ declare specific implied branches on, for example, a
+ :term:`config-project` project (which normally would not use
+ implied branches), you must set `implied-branch-matchers` as
+ well.
diff --git a/tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml b/tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml
new file mode 100644
index 0000000..dc83f9d
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml
@@ -0,0 +1,61 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ post-review: True
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ start:
+ gerrit:
+ Verified: 0
+ precedence: high
+
+- job:
+ name: base
+ parent: null
+
+- project:
+ name: common-config
+ check:
+ jobs: []
+ gate:
+ jobs:
+ - noop
+
+- project:
+ name: org/project1
+ check:
+ jobs: []
+ gate:
+ jobs:
+ - noop
+
+- project:
+ name: org/project2
+ check:
+ jobs: []
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/README b/tests/fixtures/config/pragma-multibranch/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..6c8352a
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml
@@ -0,0 +1,13 @@
+- job:
+ name: test-job1
+ run: playbooks/test-job1.yaml
+
+- job:
+ name: test-job2
+ run: playbooks/test-job2.yaml
+
+- project-template:
+ name: test-template
+ check:
+ jobs:
+ - test-job1
diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project2/README b/tests/fixtures/config/pragma-multibranch/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..748cab2
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml
@@ -0,0 +1,7 @@
+- project:
+ name: org/project2
+ templates:
+ - test-template
+ check:
+ jobs:
+ - test-job2
diff --git a/tests/fixtures/config/pragma-multibranch/main.yaml b/tests/fixtures/config/pragma-multibranch/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/pragma-multibranch/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 1f401d0..44aa966 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -1552,6 +1552,32 @@
C.messages[0],
"C should have an error reported")
+ def test_pipeline_debug(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test1
+ run: playbooks/project-test1.yaml
+ - project:
+ name: org/project
+ check:
+ debug: True
+ jobs:
+ - project-test1
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report success")
+ self.assertIn('Debug information:',
+ A.messages[0], "A should have debug info")
+
class TestInRepoJoin(ZuulTestCase):
# In this config, org/project is not a member of any pipelines, so
@@ -2232,6 +2258,115 @@
self.assertIsNone(job.branch_matcher)
+class TestPragmaMultibranch(ZuulTestCase):
+ tenant_config_file = 'config/pragma-multibranch/main.yaml'
+
+ def test_no_branch_matchers(self):
+ self.create_branch('org/project1', 'stable/pike')
+ self.create_branch('org/project2', 'stable/jewel')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable/pike'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project2', 'stable/jewel'))
+ self.waitUntilSettled()
+ # We want the jobs defined on the stable/pike branch of
+ # project1 to apply to the stable/jewel branch of project2.
+
+ # First, without the pragma line, the jobs should not run
+ # because in project1 they have branch matchers for pike, so
+ # they will not match a jewel change.
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable/jewel', 'B')
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([])
+
+ # Add a pragma line to disable implied branch matchers in
+ # project1, so that the jobs and templates apply to both
+ # branches.
+ with open(os.path.join(FIXTURE_DIR,
+ 'config/pragma-multibranch/git/',
+ 'org_project1/zuul.yaml')) as f:
+ config = f.read()
+ extra_conf = textwrap.dedent(
+ """
+ - pragma:
+ implied-branch-matchers: False
+ """)
+ config = extra_conf + config
+ file_dict = {'zuul.yaml': config}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable/pike', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+ self.waitUntilSettled()
+
+ # Now verify that when we propose a change to jewel, we get
+ # the pike/jewel jobs.
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='test-job1', result='SUCCESS', changes='1,1'),
+ dict(name='test-job2', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+ def test_supplied_branch_matchers(self):
+ self.create_branch('org/project1', 'stable/pike')
+ self.create_branch('org/project2', 'stable/jewel')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable/pike'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project2', 'stable/jewel'))
+ self.waitUntilSettled()
+ # We want the jobs defined on the stable/pike branch of
+ # project1 to apply to the stable/jewel branch of project2.
+
+ # First, without the pragma line, the jobs should not run
+ # because in project1 they have branch matchers for pike, so
+ # they will not match a jewel change.
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable/jewel', 'B')
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([])
+
+ # Add a pragma line to disable implied branch matchers in
+ # project1, so that the jobs and templates apply to both
+ # branches.
+ with open(os.path.join(FIXTURE_DIR,
+ 'config/pragma-multibranch/git/',
+ 'org_project1/zuul.yaml')) as f:
+ config = f.read()
+ extra_conf = textwrap.dedent(
+ """
+ - pragma:
+ implied-branches:
+ - stable/pike
+ - stable/jewel
+ """)
+ config = extra_conf + config
+ file_dict = {'zuul.yaml': config}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable/pike', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+ self.waitUntilSettled()
+ # Now verify that when we propose a change to jewel, we get
+ # the pike/jewel jobs.
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='test-job1', result='SUCCESS', changes='1,1'),
+ dict(name='test-job2', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
class TestBaseJobs(ZuulTestCase):
tenant_config_file = 'config/base-jobs/main.yaml'
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
index 2a4ea1d..c0ee9be 100755
--- a/tools/encrypt_secret.py
+++ b/tools/encrypt_secret.py
@@ -58,7 +58,7 @@
"to standard output.")
args = parser.parse_args()
- req = Request("%s/%s.pub" % (args.url, args.project))
+ req = Request("%s/%s.pub" % (args.url.rstrip('/'), args.project))
pubkey = urlopen(req)
if args.infile:
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 227e352..71c4ccc 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -358,6 +358,7 @@
class PragmaParser(object):
pragma = {
'implied-branch-matchers': bool,
+ 'implied-branches': to_list(str),
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
}
@@ -372,11 +373,14 @@
self.schema(conf)
bm = conf.get('implied-branch-matchers')
- if bm is None:
- return
source_context = conf['_source_context']
- source_context.implied_branch_matchers = bm
+ if bm is not None:
+ source_context.implied_branch_matchers = bm
+
+ branches = conf.get('implied-branches')
+ if branches is not None:
+ source_context.implied_branches = as_list(branches)
class NodeSetParser(object):
@@ -528,6 +532,8 @@
# If the user has set a pragma directive for this, use the
# value (if unset, the value is None).
if job.source_context.implied_branch_matchers is True:
+ if job.source_context.implied_branches is not None:
+ return job.source_context.implied_branches
return [job.source_context.branch]
elif job.source_context.implied_branch_matchers is False:
return None
@@ -543,6 +549,8 @@
if len(branches) == 1:
return None
+ if job.source_context.implied_branches is not None:
+ return job.source_context.implied_branches
return [job.source_context.branch]
@staticmethod
@@ -781,7 +789,11 @@
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
- pipeline_contents = {'queue': str, 'jobs': job_list}
+ pipeline_contents = {
+ 'queue': str,
+ 'debug': bool,
+ 'jobs': job_list,
+ }
for p in self.layout.pipelines.values():
project_template[p.name] = pipeline_contents
@@ -801,6 +813,7 @@
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
+ project_pipeline.debug = conf_pipeline.get('debug')
self.parseJobList(
conf_pipeline.get('jobs', []),
source_context, start_mark, project_pipeline.job_list)
@@ -851,7 +864,11 @@
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
- pipeline_contents = {'queue': str, 'jobs': job_list}
+ pipeline_contents = {
+ 'queue': str,
+ 'debug': bool,
+ 'jobs': job_list
+ }
for p in self.layout.pipelines.values():
project[p.name] = pipeline_contents
@@ -912,6 +929,7 @@
for pipeline in self.layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
queue_name = None
+ debug = False
# For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed.
pipeline_defined = False
@@ -924,8 +942,12 @@
implied_branch)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
+ if template_pipeline.debug is not None:
+ debug = template_pipeline.debug
if queue_name:
project_pipeline.queue_name = queue_name
+ if debug:
+ project_pipeline.debug = True
if pipeline_defined:
project_config.pipelines[pipeline.name] = project_pipeline
return project_config
diff --git a/zuul/model.py b/zuul/model.py
index e53a357..77770b7 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -641,6 +641,7 @@
self.path = path
self.trusted = trusted
self.implied_branch_matchers = None
+ self.implied_branches = None
def __str__(self):
return '%s/%s@%s' % (self.project, self.path, self.branch)
@@ -1336,6 +1337,7 @@
self.unable_to_merge = False
self.config_error = None # None or an error message string.
self.failing_reasons = []
+ self.debug_messages = []
self.merge_state = self.NEW
self.nodesets = {} # job -> nodeset
self.node_requests = {} # job -> reqs
@@ -1500,6 +1502,17 @@
def setReportedResult(self, result):
self.current_build_set.result = result
+ def debug(self, msg, indent=0):
+ ppc = self.layout.getProjectPipelineConfig(self.change.project,
+ self.pipeline)
+ if not ppc.debug:
+ return
+ if indent:
+ indent = ' ' * indent
+ else:
+ indent = ''
+ self.current_build_set.debug_messages.append(indent + msg)
+
def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and
store the resulting job tree."""
@@ -2219,6 +2232,7 @@
def __init__(self):
self.job_list = JobList()
self.queue_name = None
+ self.debug = False
self.merge_mode = None
@@ -2544,7 +2558,8 @@
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
- def collectJobs(self, jobname, change, path=None, jobs=None, stack=None):
+ def collectJobs(self, item, jobname, change, path=None, jobs=None,
+ stack=None):
if stack is None:
stack = []
if jobs is None:
@@ -2553,13 +2568,20 @@
path = []
path.append(jobname)
matched = False
+ indent = len(path) + 1
+ item.debug("Collecting job variants for {jobname}".format(
+ jobname=jobname), indent=indent)
for variant in self.getJobs(jobname):
if not variant.changeMatches(change):
self.log.debug("Variant %s did not match %s", repr(variant),
change)
+ item.debug("Variant {variant} did not match".format(
+ variant=repr(variant)), indent=indent)
continue
else:
self.log.debug("Variant %s matched %s", repr(variant), change)
+ item.debug("Variant {variant} matched".format(
+ variant=repr(variant)), indent=indent)
if not variant.isBase():
parent = variant.parent
if not jobs and parent is None:
@@ -2569,30 +2591,38 @@
if parent and parent not in path:
if parent in stack:
raise Exception("Dependency cycle in jobs: %s" % stack)
- self.collectJobs(parent, change, path, jobs, stack + [jobname])
+ self.collectJobs(item, parent, change, path, jobs,
+ stack + [jobname])
matched = True
jobs.append(variant)
if not matched:
+ self.log.debug("No matching parents for job %s and change %s",
+ jobname, change)
+ item.debug("No matching parent for {jobname}".format(
+ jobname=repr(jobname)), indent=indent)
raise NoMatchingParentError()
return jobs
def _createJobGraph(self, item, job_list, job_graph):
change = item.change
pipeline = item.pipeline
+ item.debug("Freezing job graph")
for jobname in job_list.jobs:
# This is the final job we are constructing
frozen_job = None
self.log.debug("Collecting jobs %s for %s", jobname, change)
+ item.debug("Freezing job {jobname}".format(
+ jobname=jobname), indent=1)
try:
- variants = self.collectJobs(jobname, change)
+ variants = self.collectJobs(item, jobname, change)
except NoMatchingParentError:
- self.log.debug("No matching parents for job %s and change %s",
- jobname, change)
variants = None
if not variants:
# A change must match at least one defined job variant
# (that is to say that it must match more than just
# the job that is defined in the tree).
+ item.debug("No matching variants for {jobname}".format(
+ jobname=jobname), indent=2)
continue
for variant in variants:
if frozen_job is None:
@@ -2611,12 +2641,18 @@
matched = True
self.log.debug("Pipeline variant %s matched %s",
repr(variant), change)
+ item.debug("Pipeline variant {variant} matched".format(
+ variant=repr(variant)), indent=2)
else:
self.log.debug("Pipeline variant %s did not match %s",
repr(variant), change)
+ item.debug("Pipeline variant {variant} did not match".format(
+ variant=repr(variant)), indent=2)
if not matched:
# A change must match at least one project pipeline
# job variant.
+ item.debug("No matching pipeline variants for {jobname}".
+ format(jobname=jobname), indent=2)
continue
if (frozen_job.allowed_projects and
change.project.name not in frozen_job.allowed_projects):
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 49181a7..ecf8855 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -64,6 +64,10 @@
a reporter taking free-form text."""
ret = self._getFormatter()(item, with_jobs)
+ if item.current_build_set.debug_messages:
+ debug = '\n '.join(item.current_build_set.debug_messages)
+ ret += '\nDebug information:\n ' + debug + '\n'
+
if item.pipeline.footer_message:
ret += '\n' + item.pipeline.footer_message