Avoid creating extra project schemas
Voluptuous schema creation is expensive. A recent change[1] updated
the ProjectParser to create a schema for each project stanza. This
is because we moved the instantiation of the ProjectParser class to
a point before the pipelines are parsed, and the project schema
requires knowing the pipelines.
Correct this by instantiating the ProjectParser after the pipelines
are set. With that in place, we can go back to creating only a single
project schema.
While we're at it, update some other classes to use cached schemas as
well for further performance improvements.
[1] 831ff3ebba3932c671f7e67a4ad4ed741b2cd08f
Change-Id: Ied5a1ff4a8717f4fb4fa214af57dbeba21c52afb
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index dcef666..6ec5232 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -23,6 +23,7 @@
from zuul import configloader
from zuul.lib import encryption
from zuul.lib import yamlutil as yaml
+import zuul.lib.connections
from tests.base import BaseTestCase, FIXTURE_DIR
@@ -36,6 +37,8 @@
class TestJob(BaseTestCase):
def setUp(self):
super(TestJob, self).setUp()
+ self.connections = zuul.lib.connections.ConnectionRegistry()
+ self.addCleanup(self.connections.stop)
self.connection = Dummy(connection_name='dummy_connection')
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
@@ -48,7 +51,8 @@
self.layout.addPipeline(self.pipeline)
self.queue = model.ChangeQueue(self.pipeline)
self.pcontext = configloader.ParseContext(
- None, None, self.tenant, self.layout)
+ self.connections, None, self.tenant, self.layout)
+ self.pcontext.setPipelines()
private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
with open(private_key_file, "rb") as f:
diff --git a/zuul/configloader.py b/zuul/configloader.py
index df6336d..3511f96 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -388,6 +388,8 @@
def __init__(self, pcontext):
self.log = logging.getLogger("zuul.NodeSetParser")
self.pcontext = pcontext
+ self.schema = self.getSchema(False)
+ self.anon_schema = self.getSchema(True)
def getSchema(self, anonymous=False):
node = {vs.Required('name'): to_list(str),
@@ -409,7 +411,10 @@
return vs.Schema(nodeset)
def fromYaml(self, conf, anonymous=False):
- self.getSchema(anonymous)(conf)
+ if anonymous:
+ self.anon_schema(conf)
+ else:
+ self.schema(conf)
ns = model.NodeSet(conf.get('name'), conf.get('_source_context'))
node_names = set()
group_names = set()
@@ -813,6 +818,7 @@
def __init__(self, pcontext):
self.log = logging.getLogger("zuul.ProjectTemplateParser")
self.pcontext = pcontext
+ self.schema = self.getSchema()
def getSchema(self):
project_template = {
@@ -840,7 +846,7 @@
def fromYaml(self, conf, validate=True):
if validate:
with configuration_exceptions('project-template', conf):
- self.getSchema()(conf)
+ self.schema(conf)
source_context = conf['_source_context']
project_template = model.ProjectConfig(conf['name'], source_context)
start_mark = conf['_start_mark']
@@ -884,6 +890,7 @@
def __init__(self, pcontext):
self.log = logging.getLogger("zuul.ProjectParser")
self.pcontext = pcontext
+ self.schema = self.getSchema()
def getSchema(self):
project = {
@@ -912,7 +919,7 @@
def fromYaml(self, conf_list):
for conf in conf_list:
with configuration_exceptions('project', conf):
- self.getSchema()(conf)
+ self.schema(conf)
with configuration_exceptions('project', conf_list[0]):
project_name = conf_list[0]['name']
@@ -1001,6 +1008,7 @@
def __init__(self, pcontext):
self.log = logging.getLogger("zuul.PipelineParser")
self.pcontext = pcontext
+ self.schema = self.getSchema()
def getDriverSchema(self, dtype):
methods = {
@@ -1063,7 +1071,7 @@
def fromYaml(self, conf):
with configuration_exceptions('pipeline', conf):
- self.getSchema()(conf)
+ self.schema(conf)
pipeline = model.Pipeline(conf['name'], self.pcontext.layout)
pipeline.description = conf.get('description')
@@ -1185,6 +1193,12 @@
self.secret_parser = SecretParser(self)
self.job_parser = JobParser(self)
self.semaphore_parser = SemaphoreParser(self)
+ self.project_template_parser = None
+ self.project_parser = None
+
+ def setPipelines(self):
+ # Call after pipelines are fixed in the layout to construct
+ # the project parser, which relies on them.
self.project_template_parser = ProjectTemplateParser(self)
self.project_parser = ProjectParser(self)
@@ -1616,6 +1630,7 @@
continue
layout.addPipeline(pcontext.pipeline_parser.fromYaml(
config_pipeline))
+ pcontext.setPipelines()
for config_nodeset in data.nodesets:
classes = self._getLoadClasses(tenant, config_nodeset)