Add job.nodeset parameter to supercede job.nodes
We intended to have Nodesets be convenience methods for the 'nodes'
attribute of jobs, but be identical. When nodesets grew groups,
however, job.nodes did not. Because of the additional structure
that nodesets contain (to support groups, and likely vars in the
future), we can't simply extend the existing nodes parameter.
Add a new parameter, nodeset, which expects either a string or
an embedded nodeset definition. We're using the name 'nodeset'
here because 'nodes: nodes:' is difficult to understand.
Job.nodes will be removed soon.
(Re-proposed from I714887625c41bd1220ff05cd7356fbac589389c9)
Change-Id: I6c1c1e864704ac659efae9b28b140d9b37cef9d2
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 973470d..025ea71 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -497,9 +497,10 @@
- job:
name: run-tests
parent: base
- nodes:
- - name: test-node
- label: fedora
+ nodeset:
+ nodes:
+ - name: test-node
+ label: fedora
.. attr:: job
@@ -630,12 +631,12 @@
- job:
name: run-tests
- nodes: current-release
+ nodeset: current-release
- job:
name: run-tests
branch: stable/2.0
- nodes: old-release
+ nodeset: old-release
In some cases, Zuul uses an implied value for the branch
specifier if none is supplied:
@@ -722,18 +723,19 @@
ssh_key:
key: descrypted-secret-key-data
- .. attr:: nodes
+ .. attr:: nodeset
- A list of nodes which should be supplied to the job. This
- parameter may be supplied either as a string, in which case it
- references a :ref:`nodeset` definition which appears elsewhere
- in the configuration, or a list, in which case it is interpreted
- in the same way as a Nodeset definition (in essence, it is an
- anonymous Node definition unique to this job). See the
- :ref:`nodeset` reference for the syntax to use in that case.
+ The nodes which should be supplied to the job. This parameter
+ may be supplied either as a string, in which case it references
+ a :ref:`nodeset` definition which appears elsewhere in the
+ configuration, or a dictionary, in which case it is interpreted
+ in the same way as a Nodeset definition, though the ``name``
+ attribute should be omitted (in essence, it is an anonymous
+ Nodeset definition unique to this job). See the :ref:`nodeset`
+ reference for the syntax to use in that case.
- If a job has an empty or no node definition, it will still run
- and may be able to perform actions on the Zuul executor.
+ If a job has an empty or no nodeset definition, it will still
+ run and may be able to perform actions on the Zuul executor.
.. attr:: override-branch
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index d34d5c4..67d1c70 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -98,9 +98,10 @@
- job:
parent: python27
name: check-vars
- nodes:
- - name: ubuntu-xenial
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: ubuntu-xenial
+ label: ubuntu-xenial
vars:
vartest_job: vartest_job
vartest_secret: vartest_job
@@ -112,9 +113,10 @@
- job:
parent: python27
name: check-secret-names
- nodes:
- - name: ubuntu-xenial
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: ubuntu-xenial
+ label: ubuntu-xenial
secrets:
- secret: vartest_secret
name: renamed_secret
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index 7809c5d..e5727a2 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -37,10 +37,11 @@
- job:
name: single-inventory
- nodes:
- - name: ubuntu-xenial
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: ubuntu-xenial
+ label: ubuntu-xenial
- job:
name: group-inventory
- nodes: nodeset1
+ nodeset: nodeset1
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
index 27f2fd5..273469c 100644
--- a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
@@ -17,6 +17,7 @@
- job:
name: python27
- nodes:
- - name: controller
- label: ubuntu-trusty
+ nodeset:
+ nodes:
+ - name: controller
+ label: ubuntu-trusty
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
index 2506db0..de6321d 100644
--- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -37,9 +37,10 @@
name: base
parent: null
timeout: 30
- nodes:
- - name: controller
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: controller
+ label: ubuntu-xenial
- job:
name: python27
@@ -49,9 +50,10 @@
name: python27
parent: base
branches: stable/mitaka
- nodes:
- - name: controller
- label: ubuntu-trusty
+ nodeset:
+ nodes:
+ - name: controller
+ label: ubuntu-trusty
- job:
name: python35
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 9796fe2..14f43f4 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -47,41 +47,47 @@
- job:
name: project-merge
hold-following-changes: true
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
attempts: 4
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
branches: stable
- nodes:
- - name: controller
- label: label2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
- job:
name: project-post
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- job:
name: project-test2
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project1-project2-integration
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-testfile
diff --git a/tests/fixtures/layouts/autohold.yaml b/tests/fixtures/layouts/autohold.yaml
index 515f79d..578f886 100644
--- a/tests/fixtures/layouts/autohold.yaml
+++ b/tests/fixtures/layouts/autohold.yaml
@@ -17,9 +17,10 @@
- job:
name: project-test2
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- project:
name: org/project
diff --git a/tests/fixtures/layouts/disable_at.yaml b/tests/fixtures/layouts/disable_at.yaml
index 7b1b8c8..8c24c1b 100644
--- a/tests/fixtures/layouts/disable_at.yaml
+++ b/tests/fixtures/layouts/disable_at.yaml
@@ -21,9 +21,10 @@
- job:
name: project-test1
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- project:
name: org/project
diff --git a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
index 6a92deb..bb98b57 100644
--- a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
+++ b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
@@ -13,9 +13,10 @@
- job:
name: project-post
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/fixtures/layouts/idle.yaml b/tests/fixtures/layouts/idle.yaml
index ec31408..4cc07ae 100644
--- a/tests/fixtures/layouts/idle.yaml
+++ b/tests/fixtures/layouts/idle.yaml
@@ -11,9 +11,10 @@
- job:
name: project-bitrot
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/fixtures/layouts/no-timer.yaml b/tests/fixtures/layouts/no-timer.yaml
index 3790ea7..7aaa1ed 100644
--- a/tests/fixtures/layouts/no-timer.yaml
+++ b/tests/fixtures/layouts/no-timer.yaml
@@ -29,9 +29,10 @@
- job:
name: project-bitrot
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/fixtures/layouts/repo-deleted.yaml b/tests/fixtures/layouts/repo-deleted.yaml
index 6e6c301..3a7f6b3 100644
--- a/tests/fixtures/layouts/repo-deleted.yaml
+++ b/tests/fixtures/layouts/repo-deleted.yaml
@@ -42,16 +42,18 @@
- job:
name: project-test1
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
branches: stable
- nodes:
- - name: controller
- label: label2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
- job:
name: project-test2
diff --git a/tests/fixtures/layouts/smtp.yaml b/tests/fixtures/layouts/smtp.yaml
index 5ea75ce..0654448 100644
--- a/tests/fixtures/layouts/smtp.yaml
+++ b/tests/fixtures/layouts/smtp.yaml
@@ -48,16 +48,18 @@
- job:
name: project-test1
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
branches: stable
- nodes:
- - name: controller
- label: label2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
- job:
name: project-test2
diff --git a/tests/fixtures/layouts/timer.yaml b/tests/fixtures/layouts/timer.yaml
index e1c4e77..8c0cc2b 100644
--- a/tests/fixtures/layouts/timer.yaml
+++ b/tests/fixtures/layouts/timer.yaml
@@ -30,9 +30,10 @@
- job:
name: project-bitrot
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 2248aa9..c457ff0 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -190,10 +190,12 @@
'timeout': 30,
'pre-run': 'base-pre',
'post-run': 'base-post',
- 'nodes': [{
- 'name': 'controller',
- 'label': 'base',
- }],
+ 'nodeset': {
+ 'nodes': [{
+ 'name': 'controller',
+ 'label': 'base',
+ }],
+ },
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(tenant, layout, {
@@ -203,10 +205,12 @@
'parent': 'base',
'pre-run': 'py27-pre',
'post-run': ['py27-post-a', 'py27-post-b'],
- 'nodes': [{
- 'name': 'controller',
- 'label': 'new',
- }],
+ 'nodeset': {
+ 'nodes': [{
+ 'name': 'controller',
+ 'label': 'new',
+ }],
+ },
'timeout': 40,
})
layout.addJob(python27)
@@ -220,10 +224,12 @@
'pre-run': 'py27-diablo-pre',
'run': 'py27-diablo',
'post-run': 'py27-diablo-post',
- 'nodes': [{
- 'name': 'controller',
- 'label': 'old',
- }],
+ 'nodeset': {
+ 'nodes': [{
+ 'name': 'controller',
+ 'label': 'old',
+ }],
+ },
'timeout': 50,
})
layout.addJob(python27diablo)
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index d55ff92..9a10e9d 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -1407,19 +1407,20 @@
"""
- job:
name: test-job
- nodes:
- - name: node01
- label: fake
- - name: node02
- label: fake
- - name: node03
- label: fake
- - name: node04
- label: fake
- - name: node05
- label: fake
- - name: node06
- label: fake
+ nodeset:
+ nodes:
+ - name: node01
+ label: fake
+ - name: node02
+ label: fake
+ - name: node03
+ label: fake
+ - name: node04
+ label: fake
+ - name: node05
+ label: fake
+ - name: node06
+ label: fake
""")
file_dict = {'.zuul.yaml': in_repo_conf}
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 13fc310..62439c4 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -287,7 +287,7 @@
class NodeSetParser(object):
@staticmethod
- def getSchema():
+ def getSchema(anonymous=False):
node = {vs.Required('name'): str,
vs.Required('label'): str,
}
@@ -296,19 +296,20 @@
vs.Required('nodes'): to_list(str),
}
- nodeset = {vs.Required('name'): str,
- vs.Required('nodes'): to_list(node),
+ nodeset = {vs.Required('nodes'): to_list(node),
'groups': to_list(group),
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
}
+ if not anonymous:
+ nodeset[vs.Required('name')] = str
return vs.Schema(nodeset)
@staticmethod
- def fromYaml(layout, conf):
- NodeSetParser.getSchema()(conf)
- ns = model.NodeSet(conf['name'])
+ def fromYaml(conf, anonymous=False):
+ NodeSetParser.getSchema(anonymous)(conf)
+ ns = model.NodeSet(conf.get('name'))
node_names = set()
group_names = set()
for conf_node in as_list(conf['nodes']):
@@ -391,6 +392,8 @@
'secrets': to_list(vs.Any(secret, str)),
'irrelevant-files': to_list(str),
'nodes': vs.Any([node], str),
+ # validation happens in NodeSetParser
+ 'nodeset': vs.Any(dict, str),
'timeout': int,
'attempts': int,
'pre-run': to_list(str),
@@ -569,7 +572,18 @@
a = k.replace('-', '_')
if k in conf:
setattr(job, a, conf[k])
- if 'nodes' in conf:
+ if 'nodeset' in conf:
+ conf_nodeset = conf['nodeset']
+ if isinstance(conf_nodeset, str):
+ # This references an existing named nodeset in the layout.
+ ns = layout.nodesets[conf_nodeset]
+ else:
+ ns = NodeSetParser.fromYaml(conf_nodeset, anonymous=True)
+ if tenant.max_nodes_per_job != -1 and \
+ len(ns) > tenant.max_nodes_per_job:
+ raise MaxNodeError(job, tenant)
+ job.nodeset = ns
+ elif 'nodes' in conf:
conf_nodes = conf['nodes']
if isinstance(conf_nodes, str):
# This references an existing named nodeset in the layout.
@@ -1448,7 +1462,7 @@
continue
with configuration_exceptions('nodeset', config_nodeset):
layout.addNodeSet(NodeSetParser.fromYaml(
- layout, config_nodeset))
+ config_nodeset))
for config_secret in data.secrets:
classes = TenantParser._getLoadClasses(tenant, config_secret)