Merge "Limit concurrency in zuul-executor under load" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index 2eadd4f..a997b05 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -77,5 +77,5 @@
- zuul-stream-functional
post:
jobs:
- - publish-openstack-python-docs-infra
+ - publish-openstack-sphinx-docs-infra
- publish-openstack-python-branch-tarball
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 025ea71..f55fb4f 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -653,10 +653,22 @@
configuration, Zuul reads the ``master`` branch of a given
project first, then other branches in alphabetical order.
+ * In the case of a job variant defined within a :ref:`project`,
+ if the project definition is in a :term:`config-project`, no
+ implied branch specifier is used. If it appears in an
+ :term:`untrusted-project`, with no branch specifier, the
+ branch containing the project definition is used as an implied
+ branch specifier.
+
+ * In the case of a job variant defined within a
+ :ref:`project-template`, if no branch specifier appears, the
+ implied branch specifier for the :ref:`project` definition which
+ uses the project-template will be used.
+
* Any further job variants other than the reference definition
in an untrusted-project will, if they do not have a branch
- specifier, will have an implied branch specifier for the
- current branch applied.
+ specifier, have an implied branch specifier for the current
+ branch applied.
This allows for the very simple and expected workflow where if a
project defines a job on the ``master`` branch with no branch
diff --git a/tests/fixtures/config/templated-project/git/untrusted-config/zuul.d/project.yaml b/tests/fixtures/config/templated-project/git/untrusted-config/zuul.d/project.yaml
new file mode 100644
index 0000000..6d1892c
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/untrusted-config/zuul.d/project.yaml
@@ -0,0 +1,4 @@
+- project:
+ name: untrusted-config
+ templates:
+ - test-one-and-two
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml b/tests/fixtures/config/templated-project/git/untrusted-config/zuul.d/templates.yaml
similarity index 100%
rename from tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml
rename to tests/fixtures/config/templated-project/git/untrusted-config/zuul.d/templates.yaml
diff --git a/tests/fixtures/config/templated-project/main.yaml b/tests/fixtures/config/templated-project/main.yaml
index e59b396..bb59838 100644
--- a/tests/fixtures/config/templated-project/main.yaml
+++ b/tests/fixtures/config/templated-project/main.yaml
@@ -5,5 +5,6 @@
config-projects:
- common-config
untrusted-projects:
+ - untrusted-config
- org/templated-project
- org/layered-project
diff --git a/tests/print_layout.py b/tests/print_layout.py
index a295886..055270f 100644
--- a/tests/print_layout.py
+++ b/tests/print_layout.py
@@ -59,6 +59,14 @@
if fn in ['zuul.yaml', '.zuul.yaml']:
print_file('File: ' + os.path.join(gitrepo, fn),
os.path.join(reporoot, fn))
+ for subdir in ['.zuul.d', 'zuul.d']:
+ zuuld = os.path.join(reporoot, subdir)
+ if not os.path.exists(zuuld):
+ continue
+ filenames = os.listdir(zuuld)
+ for fn in filenames:
+ print_file('File: ' + os.path.join(gitrepo, subdir, fn),
+ os.path.join(zuuld, fn))
if __name__ == '__main__':
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 7f8c0a3..9e5e36c 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -4880,6 +4880,42 @@
self.assertEqual(self.getJobFromHistory('project-test6').result,
'SUCCESS')
+ def test_unimplied_branch_matchers(self):
+ # This tests that there are no implied branch matchers added
+ # by project templates.
+ self.create_branch('org/layered-project', 'stable')
+
+ A = self.fake_gerrit.addFakeChange(
+ 'org/layered-project', 'stable', 'A')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+ print(self.getJobFromHistory('project-test1').
+ parameters['zuul']['_inheritance_path'])
+
+ def test_implied_branch_matchers(self):
+ # This tests that there is an implied branch matcher when a
+ # template is used on an in-repo project pipeline definition.
+ self.create_branch('untrusted-config', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'untrusted-config', 'stable'))
+ self.waitUntilSettled()
+
+ A = self.fake_gerrit.addFakeChange(
+ 'untrusted-config', 'stable', 'A')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+ print(self.getJobFromHistory('project-test1').
+ parameters['zuul']['_inheritance_path'])
+
class TestSchedulerSuccessURL(ZuulTestCase):
tenant_config_file = 'config/success-url/main.yaml'
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index a5db9a6..0321d69 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -25,6 +25,7 @@
import traceback
yappi = extras.try_import('yappi')
+objgraph = extras.try_import('objgraph')
from zuul.ansible import logconfig
import zuul.lib.connections
@@ -54,6 +55,11 @@
log.debug(yappi_out.getvalue())
yappi_out.close()
yappi.clear_stats()
+ if objgraph:
+ objgraph_out = io.StringIO()
+ objgraph.show_growth(limit=100, file=objgraph_out)
+ log.debug(objgraph_out.getvalue())
+ objgraph_out.close()
signal.signal(signal.SIGUSR2, stack_dump_handler)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 97dfd21..c1a65be 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -456,19 +456,22 @@
# the reference definition of this job, and this is a project
# repo, add an implicit branch matcher for this branch
# (assuming there are no explicit branch matchers). But only
- # for top-level job definitions and variants.
- # Project-pipeline job variants should more closely attach to
- # their branch if they appear in a project-repo.
+ # for top-level job definitions and variants. Never for
+ # project-templates. They, and in-project project-pipeline
+ # job variants, should more closely attach to their branch if
+ # they appear in a project-repo. That's handled in the
+ # ProjectParser.
if (reference and
reference.source_context and
reference.source_context.branch != job.source_context.branch):
- same_context = False
+ same_branch = False
else:
- same_context = True
+ same_branch = True
if (job.source_context and
(not job.source_context.trusted) and
- ((not same_context) or project_pipeline)):
+ (not project_pipeline) and
+ (not same_branch)):
return [job.source_context.branch]
return None
@@ -486,7 +489,7 @@
job = model.Job(conf['name'])
job.source_context = conf.get('_source_context')
- job.source_line = conf.get('_start_mark').line +1
+ job.source_line = conf.get('_start_mark').line + 1
is_variant = layout.hasJob(conf['name'])
if 'parent' in conf:
@@ -669,10 +672,7 @@
if (not branches) and ('branches' in conf):
branches = as_list(conf['branches'])
if branches:
- matchers = []
- for branch in branches:
- matchers.append(change_matcher.BranchMatcher(branch))
- job.branch_matcher = change_matcher.MatchAny(matchers)
+ job.setBranchMatcher(branches)
if 'files' in conf:
matchers = []
for fn in as_list(conf['files']):
@@ -730,8 +730,12 @@
return vs.Schema(project_template)
@staticmethod
- def fromYaml(tenant, layout, conf):
- with configuration_exceptions('project or project-template', conf):
+ def fromYaml(tenant, layout, conf, template):
+ if template:
+ project_or_template = 'project-template'
+ else:
+ project_or_template = 'project'
+ with configuration_exceptions(project_or_template, conf):
ProjectTemplateParser.getSchema(layout)(conf)
# Make a copy since we modify this later via pop
conf = copy.deepcopy(conf)
@@ -747,12 +751,13 @@
project_pipeline.queue_name = conf_pipeline.get('queue')
ProjectTemplateParser._parseJobList(
tenant, layout, conf_pipeline.get('jobs', []),
- source_context, start_mark, project_pipeline.job_list)
+ source_context, start_mark, project_pipeline.job_list,
+ template)
return project_template
@staticmethod
def _parseJobList(tenant, layout, conf, source_context,
- start_mark, job_list):
+ start_mark, job_list, template):
for conf_job in conf:
if isinstance(conf_job, str):
attrs = dict(name=conf_job)
@@ -815,6 +820,7 @@
configs = []
for conf in conf_list:
+ implied_branch = None
with configuration_exceptions('project', conf):
if not conf['_source_context'].trusted:
if project != conf['_source_context'].project:
@@ -828,13 +834,18 @@
# all of the templates, including the newly parsed
# one, in order.
project_template = ProjectTemplateParser.fromYaml(
- tenant, layout, conf)
+ tenant, layout, conf, template=False)
+ # If this project definition is in a place where it
+ # should get implied branch matchers, set it.
+ if (not conf['_source_context'].trusted):
+ implied_branch = conf['_source_context'].branch
for name in conf_templates:
if name not in layout.project_templates:
raise TemplateNotFoundError(name)
- configs.extend([layout.project_templates[name]
+ configs.extend([(layout.project_templates[name],
+ implied_branch)
for name in conf_templates])
- configs.append(project_template)
+ configs.append((project_template, implied_branch))
# Set the following values to the first one that we
# find and ignore subsequent settings.
mode = conf.get('merge-mode')
@@ -855,12 +866,13 @@
# For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed.
pipeline_defined = False
- for template in configs:
+ for (template, implied_branch) in configs:
if pipeline.name in template.pipelines:
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_list.inheritFrom(
- template_pipeline.job_list)
+ template_pipeline.job_list,
+ implied_branch)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if queue_name:
@@ -1520,7 +1532,7 @@
if 'project-template' not in classes:
continue
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
- tenant, layout, config_template))
+ tenant, layout, config_template, template=True))
for config_projects in data.projects.values():
# Unlike other config classes, we expect multiple project
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 0e6544b..2ca69fc 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -445,7 +445,9 @@
def lookForLostBuilds(self):
self.log.debug("Looking for lost builds")
- for build in self.builds.values():
+ # Construct a list from the values iterator to protect from it changing
+ # out from underneath us.
+ for build in list(self.builds.values()):
if build.result:
# The build has finished, it will be removed
continue
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 32c0523..e41d6b7 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -1547,6 +1547,13 @@
config.write('display_args_to_stdout = %s\n' %
str(not jobdir_playbook.secrets_content))
+ # Increase the internal poll interval of ansible.
+ # The default interval of 0.001s is optimized for interactive
+ # ui at the expense of CPU load. As we have a non-interactive
+ # automation use case a longer poll interval is more suitable
+ # and reduces CPU load of the ansible process.
+ config.write('internal_poll_interval = 0.01\n')
+
config.write('[ssh_connection]\n')
# NB: when setting pipelining = True, keep_remote_files
# must be False (the default). Otherwise it apparently
diff --git a/zuul/model.py b/zuul/model.py
index eff0ae3..4d40b6c 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -23,6 +23,8 @@
import urllib.parse
import textwrap
+from zuul import change_matcher
+
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
MERGER_CHERRY_PICK = 3 # "git cherry-pick"
@@ -532,7 +534,11 @@
@property
def priority(self):
- return PRIORITY_MAP[self.build_set.item.pipeline.precedence]
+ if self.build_set:
+ precedence = self.build_set.item.pipeline.precedence
+ else:
+ precedence = PRECEDENCE_NORMAL
+ return PRIORITY_MAP[precedence]
@property
def fulfilled(self):
@@ -911,6 +917,13 @@
if changed:
self.roles = tuple(newroles)
+ def setBranchMatcher(self, branches):
+ # Set the branch matcher to match any of the supplied branches
+ matchers = []
+ for branch in branches:
+ matchers.append(change_matcher.BranchMatcher(branch))
+ self.branch_matcher = change_matcher.MatchAny(matchers)
+
def updateVariables(self, other_vars):
v = self.variables
Job._deepUpdate(v, other_vars)
@@ -1038,14 +1051,14 @@
else:
self.jobs[job.name] = [job]
- def inheritFrom(self, other):
+ def inheritFrom(self, other, implied_branch):
for jobname, jobs in other.jobs.items():
- if jobname in self.jobs:
- self.jobs[jobname].extend(jobs)
- else:
- # Be sure to make a copy here since this list may be
- # modified.
- self.jobs[jobname] = jobs[:]
+ joblist = self.jobs.setdefault(jobname, [])
+ for job in jobs:
+ if not job.branch_matcher and implied_branch:
+ job = job.copy()
+ job.setBranchMatcher([implied_branch])
+ joblist.append(job)
class JobGraph(object):
@@ -2113,6 +2126,7 @@
def __init__(self, name):
self.name = name
self.merge_mode = None
+ # The default branch for the project (usually master).
self.default_branch = None
self.pipelines = {}
self.private_key_file = None