Remove project.unparsed_config
This holds the entire configuration for a project as a cache so we
don't need to reload everything on a config change, but so does
project.unparsed_branch_config. The latter just does it in smaller
chunks.
We started with only unparsed_config, then added unparsed_branch_config
but never fully replaced unparsed_config. That allowed for a small
gain of speed on rebuilding a configuration at a relatively larger
cost in memory.
Instead, only keep one copy of the cached config, and rebuild it
from individual branches when required.
Change-Id: I7b13c3fa288a19026776069c5e19f0b098b0b317
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 4f51c45..270b91c 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1294,7 +1294,7 @@
# an old_tenant object, however, we may be doing so because of
# a branch creation event, so if we don't have any cached
# data, query the branches again as well.
- if old_tenant and tpc.project.unparsed_config:
+ if old_tenant and tpc.project.unparsed_branch_config:
branches = old_tenant.getProjectBranches(tpc.project)[:]
else:
branches = sorted(tpc.project.source.getProjectBranches(
@@ -1444,10 +1444,6 @@
cached, tenant):
config_projects_config = model.UnparsedTenantConfig()
untrusted_projects_config = model.UnparsedTenantConfig()
- # project -> config; these will replace
- # project.unparsed_config if this method succesfully
- # completes
- new_project_unparsed_config = {}
# project -> branch -> config; these will replace
# project.unparsed_branch_config if this method succesfully
# completes
@@ -1460,18 +1456,25 @@
# data and is inserted in the ordered jobs list for later
# processing.
class CachedDataJob(object):
- def __init__(self, config_project, project):
+ def __init__(self, config_project, project, branch):
self.config_project = config_project
self.project = project
+ self.branch = branch
for project in config_projects:
# If we have cached data (this is a reconfiguration) use it.
- if cached and project.unparsed_config:
- jobs.append(CachedDataJob(True, project))
+ if cached and project.unparsed_branch_config:
+ # Note: this should only be one branch (master), as
+ # that's all we will initially load below in the
+ # un-cached case.
+ for branch in project.unparsed_branch_config.keys():
+ jobs.append(CachedDataJob(True, project, branch))
continue
# Otherwise, prepare an empty unparsed config object to
# hold cached data later.
- new_project_unparsed_config[project] = model.UnparsedTenantConfig()
+ new_project_unparsed_branch_config[project] = {}
+ new_project_unparsed_branch_config[project]['master'] = \
+ model.UnparsedTenantConfig()
# Get main config files. These files are permitted the
# full range of configuration.
job = self.merger.getFiles(
@@ -1490,12 +1493,12 @@
if not tpc.load_classes:
continue
# If we have cached data (this is a reconfiguration) use it.
- if cached and project.unparsed_config:
- jobs.append(CachedDataJob(False, project))
+ if cached and project.unparsed_branch_config:
+ for branch in project.unparsed_branch_config.keys():
+ jobs.append(CachedDataJob(False, project, branch))
continue
# Otherwise, prepare an empty unparsed config object to
# hold cached data later.
- new_project_unparsed_config[project] = model.UnparsedTenantConfig()
new_project_unparsed_branch_config[project] = {}
# Get in-project-repo config files which have a restricted
# set of options.
@@ -1527,10 +1530,10 @@
(job.project,))
if job.config_project:
config_projects_config.extend(
- job.project.unparsed_config)
+ job.project.unparsed_branch_config[job.branch])
else:
untrusted_projects_config.extend(
- job.project.unparsed_config)
+ job.project.unparsed_branch_config[job.branch])
continue
self.log.debug("Waiting for cat job %s" % (job,))
job.wait()
@@ -1567,16 +1570,10 @@
incdata = self.loadUntrustedProjectLayout(
job.files[fn], source_context)
untrusted_projects_config.extend(incdata)
- new_project_unparsed_config[project].extend(
- incdata)
- if branch in new_project_unparsed_branch_config.get(
- project, {}):
- new_project_unparsed_branch_config[project][branch].\
- extend(incdata)
+ new_project_unparsed_branch_config[project][branch].\
+ extend(incdata)
# Now that we've sucessfully loaded all of the configuration,
# cache the unparsed data on the project objects.
- for project, data in new_project_unparsed_config.items():
- project.unparsed_config = data
for project, branch_config in \
new_project_unparsed_branch_config.items():
project.unparsed_branch_config = branch_config
@@ -1810,10 +1807,7 @@
# If there is no files entry at all for this
# project-branch, then use the cached config.
if files_entry is None:
- if trusted:
- incdata = project.unparsed_config
- else:
- incdata = project.unparsed_branch_config.get(branch)
+ incdata = project.unparsed_branch_config.get(branch)
if incdata:
config.extend(incdata)
continue
diff --git a/zuul/model.py b/zuul/model.py
index d0c66e8..763eb66 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -349,7 +349,6 @@
# when deciding whether to enqueue their changes
# TODOv3 (jeblair): re-add support for foreign projects if needed
self.foreign = foreign
- self.unparsed_config = None
self.unparsed_branch_config = {} # branch -> UnparsedTenantConfig
def __str__(self):
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a195b09..083cb12 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -569,7 +569,6 @@
# If a change landed to a project, clear out the cached
# config before reconfiguring.
for project in event.projects:
- project.unparsed_config = None
project.unparsed_branch_config = {}
old_tenant = self.abide.tenants[event.tenant_name]
loader = configloader.ConfigLoader(