Merge "Forward-port v2.5 Ansible launcher improvements" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index 552bdd6..6092626 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -541,14 +541,21 @@
class FakeBuild(object):
log = logging.getLogger("zuul.test")
- def __init__(self, launch_server, job, node):
+ def __init__(self, launch_server, job):
self.daemon = True
self.launch_server = launch_server
self.job = job
self.jobdir = None
self.uuid = job.unique
- self.node = node
self.parameters = json.loads(job.arguments)
+ # TODOv3(jeblair): self.node is really "the image of the node
+ # assigned". We should rename it (self.node_image?) if we
+ # keep using it like this, or we may end up exposing more of
+ # the complexity around multi-node jobs here
+ # (self.nodes[0].image?)
+ self.node = None
+ if len(self.parameters.get('nodes')) == 1:
+ self.node = self.parameters['nodes'][0]['image']
self.unique = self.parameters['ZUUL_UUID']
self.name = self.parameters['job']
self.wait_condition = threading.Condition()
@@ -707,8 +714,7 @@
(regex, len(self.running_builds)))
def launchJob(self, job):
- node = None
- build = FakeBuild(self, job, node)
+ build = FakeBuild(self, job)
job.build = build
self.running_builds.append(build)
self.job_builds[job.unique] = build
@@ -1477,7 +1483,7 @@
self.log.error("No running builds")
raise
- def assertHistory(self, history):
+ def assertHistory(self, history, ordered=True):
"""Assert that the completed builds are as described.
The list of completed builds is examined and must match
@@ -1488,14 +1494,37 @@
history, and each element of the dictionary must match the
corresponding attribute of the build.
+ :arg bool ordered: If true, the history must match the order
+ supplied, if false, the builds are permitted to have
+ arrived in any order.
+
"""
+ def matches(history_item, item):
+ for k, v in item.items():
+ if getattr(history_item, k) != v:
+ return False
+ return True
try:
self.assertEqual(len(self.history), len(history))
- for i, d in enumerate(history):
- for k, v in d.items():
- self.assertEqual(
- getattr(self.history[i], k), v,
- "Element %i in history does not match" % (i,))
+ if ordered:
+ for i, d in enumerate(history):
+ if not matches(self.history[i], d):
+ raise Exception(
+ "Element %i in history does not match" % (i,))
+ else:
+ unseen = self.history[:]
+ for i, d in enumerate(history):
+ found = False
+ for unseen_item in unseen:
+ if matches(unseen_item, d):
+ found = True
+ unseen.remove(unseen_item)
+ break
+ if not found:
+ raise Exception("No match found for element %i "
+ "in history" % (i,))
+ if unseen:
+ raise Exception("Unexpected items in history")
except Exception:
for build in self.history:
self.log.error("Completed build: %s" % build)
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
index 785f8a5..4a653f6 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
@@ -21,6 +21,12 @@
verified: 0
precedence: high
+- nodeset:
+ name: nodeset1
+ nodes:
+ - name: controller
+ image: controller-image
+
- job:
name:
project1-test1
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
index c6127ca..7c79720 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
@@ -21,6 +21,12 @@
verified: 0
precedence: high
+- nodeset:
+ name: nodeset1
+ nodes:
+ - name: controller
+ image: controller-image
+
- job:
name:
project2-test1
diff --git a/tests/fixtures/config/openstack/git/openstack_keystone/README b/tests/fixtures/config/openstack/git/openstack_keystone/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/openstack_keystone/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/openstack/git/openstack_nova/README b/tests/fixtures/config/openstack/git/openstack_nova/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/openstack_nova/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
new file mode 100644
index 0000000..9c2231a
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -0,0 +1,88 @@
+# Pipeline definitions
+
+- pipeline:
+ name: check
+ manager: independent
+ success-message: Build succeeded (check).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+# Job definitions
+
+- job:
+ name: base
+ timeout: 30
+ nodes:
+ - name: controller
+ image: ubuntu-xenial
+
+- job:
+ name: python27
+ parent: base
+
+- job:
+ name: python27
+ parent: base
+ branches: stable/mitaka
+ nodes:
+ - name: controller
+ image: ubuntu-trusty
+
+- job:
+ name: python35
+ parent: base
+
+- project-template:
+ name: python-jobs
+ gate:
+ jobs:
+ - python27
+ - python35
+
+# Project definitions
+
+- project:
+ name: openstack/nova
+ templates:
+ - python-jobs
+ gate:
+ queue: integrated
+
+- project:
+ name: openstack/keystone
+ templates:
+ - python-jobs
+ gate:
+ queue: integrated
diff --git a/tests/fixtures/config/openstack/main.yaml b/tests/fixtures/config/openstack/main.yaml
new file mode 100644
index 0000000..95a0952
--- /dev/null
+++ b/tests/fixtures/config/openstack/main.yaml
@@ -0,0 +1,6 @@
+- tenant:
+ name: openstack
+ source:
+ gerrit:
+ config-repos:
+ - project-config
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 3a88863..01de2aa 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -42,6 +42,16 @@
- job:
name: project-test1
+ nodes:
+ - name: controller
+ image: image1
+
+- job:
+ name: project-test1
+ branches: stable
+ nodes:
+ - name: controller
+ image: image2
- job:
name: project-test2
@@ -54,3 +64,23 @@
jobs:
- project-test1
- project-test2
+
+- project:
+ name: org/project1
+ gate:
+ queue: integrated
+ jobs:
+ - project-merge:
+ jobs:
+ - project-test1
+ - project-test2
+
+- project:
+ name: org/project2
+ gate:
+ queue: integrated
+ jobs:
+ - project-merge:
+ jobs:
+ - project-test1
+ - project-test2
diff --git a/tests/fixtures/config/single-tenant/git/org_project1/README b/tests/fixtures/config/single-tenant/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/single-tenant/git/org_project2/README b/tests/fixtures/config/single-tenant/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/test_openstack.py b/tests/test_openstack.py
new file mode 100644
index 0000000..175b4bd
--- /dev/null
+++ b/tests/test_openstack.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from tests.base import AnsibleZuulTestCase
+
+logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(name)-32s '
+ '%(levelname)-8s %(message)s')
+
+
+class TestOpenStack(AnsibleZuulTestCase):
+ # A temporary class to experiment with how openstack can use
+ # Zuulv3
+
+ tenant_config_file = 'config/openstack/main.yaml'
+
+ def test_nova_master(self):
+ A = self.fake_gerrit.addFakeChange('openstack/nova', 'master', 'A')
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('python27').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('python35').result,
+ 'SUCCESS')
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2,
+ "A should report start and success")
+ self.assertEqual(self.getJobFromHistory('python27').node,
+ 'ubuntu-xenial')
+
+ def test_nova_mitaka(self):
+ self.create_branch('openstack/nova', 'stable/mitaka')
+ A = self.fake_gerrit.addFakeChange('openstack/nova',
+ 'stable/mitaka', 'A')
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('python27').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('python35').result,
+ 'SUCCESS')
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2,
+ "A should report start and success")
+ self.assertEqual(self.getJobFromHistory('python27').node,
+ 'ubuntu-trusty')
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 51ece68..d912ff3 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -61,6 +61,8 @@
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
+ self.assertEqual(self.getJobFromHistory('project-test1').node,
+ 'image1')
# TODOv3(jeblair): we may want to report stats by tenant (also?).
self.assertReportedStat('gerrit.event.comment-added', value='1|c')
@@ -88,6 +90,25 @@
self.assertReportedStat('zuul.pipeline.check.current_changes',
value='0|g')
+ def test_job_branch(self):
+ "Test the correct variant of a job runs on a branch"
+ self.create_branch('org/project', 'stable')
+ A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('project-test2').result,
+ 'SUCCESS')
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2,
+ "A should report start and success")
+ self.assertIn('gate', A.messages[1],
+ "A should transit gate")
+ self.assertEqual(self.getJobFromHistory('project-test1').node,
+ 'image2')
+
@skip("Disabled for early v3 development")
def test_duplicate_pipelines(self):
"Test that a change matching multiple pipelines works"
@@ -241,14 +262,13 @@
dict(name='project-merge', result='SUCCESS', changes='2,1'),
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='2,1'),
- ])
+ ], ordered=False)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
- @skip("Disabled for early v3 development")
def test_independent_queues(self):
"Test that changes end up in the right queues"
@@ -267,28 +287,43 @@
self.waitUntilSettled()
# There should be one merge job at the head of each queue running
- self.assertEqual(len(self.builds), 2)
- self.assertEqual(self.builds[0].name, 'project-merge')
- self.assertTrue(self.job_has_changes(self.builds[0], A))
- self.assertEqual(self.builds[1].name, 'project1-merge')
- self.assertTrue(self.job_has_changes(self.builds[1], B))
+ self.assertBuilds([
+ dict(name='project-merge', changes='1,1'),
+ dict(name='project-merge', changes='2,1'),
+ ])
# Release the current merge builds
- self.launch_server.release('.*-merge')
+ self.builds[0].release()
+ self.waitUntilSettled()
+ self.builds[0].release()
self.waitUntilSettled()
# Release the merge job for project2 which is behind project1
self.launch_server.release('.*-merge')
self.waitUntilSettled()
# All the test builds should be running:
- # project1 (3) + project2 (3) + project (2) = 8
- self.assertEqual(len(self.builds), 8)
+ self.assertBuilds([
+ dict(name='project-test1', changes='1,1'),
+ dict(name='project-test2', changes='1,1'),
+ dict(name='project-test1', changes='2,1'),
+ dict(name='project-test2', changes='2,1'),
+ dict(name='project-test1', changes='2,1 3,1'),
+ dict(name='project-test2', changes='2,1 3,1'),
+ ])
- self.launch_server.release()
- self.waitUntilSettled()
- self.assertEqual(len(self.builds), 0)
+ self.orderedRelease()
+ self.assertHistory([
+ dict(name='project-merge', result='SUCCESS', changes='1,1'),
+ dict(name='project-merge', result='SUCCESS', changes='2,1'),
+ dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
+ dict(name='project-test1', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ dict(name='project-test1', result='SUCCESS', changes='2,1'),
+ dict(name='project-test2', result='SUCCESS', changes='2,1'),
+ dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
+ dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
+ ])
- self.assertEqual(len(self.history), 11)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
@@ -296,7 +331,6 @@
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
- @skip("Disabled for early v3 development")
def test_failed_change_at_head(self):
"Test that if a change at the head fails, jobs behind it are canceled"
@@ -316,9 +350,9 @@
self.waitUntilSettled()
- self.assertEqual(len(self.builds), 1)
- self.assertEqual(self.builds[0].name, 'project-merge')
- self.assertTrue(self.job_has_changes(self.builds[0], A))
+ self.assertBuilds([
+ dict(name='project-merge', changes='1,1'),
+ ])
self.launch_server.release('.*-merge')
self.waitUntilSettled()
@@ -327,27 +361,84 @@
self.launch_server.release('.*-merge')
self.waitUntilSettled()
- self.assertEqual(len(self.builds), 6)
- self.assertEqual(self.builds[0].name, 'project-test1')
- self.assertEqual(self.builds[1].name, 'project-test2')
- self.assertEqual(self.builds[2].name, 'project-test1')
- self.assertEqual(self.builds[3].name, 'project-test2')
- self.assertEqual(self.builds[4].name, 'project-test1')
- self.assertEqual(self.builds[5].name, 'project-test2')
+ self.assertBuilds([
+ dict(name='project-test1', changes='1,1'),
+ dict(name='project-test2', changes='1,1'),
+ dict(name='project-test1', changes='1,1 2,1'),
+ dict(name='project-test2', changes='1,1 2,1'),
+ dict(name='project-test1', changes='1,1 2,1 3,1'),
+ dict(name='project-test2', changes='1,1 2,1 3,1'),
+ ])
self.release(self.builds[0])
self.waitUntilSettled()
# project-test2, project-merge for B
- self.assertEqual(len(self.builds), 2)
- self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
+ self.assertBuilds([
+ dict(name='project-test2', changes='1,1'),
+ dict(name='project-merge', changes='2,1'),
+ ])
+ # Unordered history comparison because the aborts can finish
+ # in any order.
+ self.assertHistory([
+ dict(name='project-merge', result='SUCCESS',
+ changes='1,1'),
+ dict(name='project-merge', result='SUCCESS',
+ changes='1,1 2,1'),
+ dict(name='project-merge', result='SUCCESS',
+ changes='1,1 2,1 3,1'),
+ dict(name='project-test1', result='FAILURE',
+ changes='1,1'),
+ dict(name='project-test1', result='ABORTED',
+ changes='1,1 2,1'),
+ dict(name='project-test2', result='ABORTED',
+ changes='1,1 2,1'),
+ dict(name='project-test1', result='ABORTED',
+ changes='1,1 2,1 3,1'),
+ dict(name='project-test2', result='ABORTED',
+ changes='1,1 2,1 3,1'),
+ ], ordered=False)
- self.launch_server.hold_jobs_in_build = False
- self.launch_server.release()
+ self.launch_server.release('.*-merge')
self.waitUntilSettled()
+ self.launch_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.orderedRelease()
- self.assertEqual(len(self.builds), 0)
- self.assertEqual(len(self.history), 15)
+ self.assertBuilds([])
+ self.assertHistory([
+ dict(name='project-merge', result='SUCCESS',
+ changes='1,1'),
+ dict(name='project-merge', result='SUCCESS',
+ changes='1,1 2,1'),
+ dict(name='project-merge', result='SUCCESS',
+ changes='1,1 2,1 3,1'),
+ dict(name='project-test1', result='FAILURE',
+ changes='1,1'),
+ dict(name='project-test1', result='ABORTED',
+ changes='1,1 2,1'),
+ dict(name='project-test2', result='ABORTED',
+ changes='1,1 2,1'),
+ dict(name='project-test1', result='ABORTED',
+ changes='1,1 2,1 3,1'),
+ dict(name='project-test2', result='ABORTED',
+ changes='1,1 2,1 3,1'),
+ dict(name='project-merge', result='SUCCESS',
+ changes='2,1'),
+ dict(name='project-merge', result='SUCCESS',
+ changes='2,1 3,1'),
+ dict(name='project-test2', result='SUCCESS',
+ changes='1,1'),
+ dict(name='project-test1', result='SUCCESS',
+ changes='2,1'),
+ dict(name='project-test2', result='SUCCESS',
+ changes='2,1'),
+ dict(name='project-test1', result='SUCCESS',
+ changes='2,1 3,1'),
+ dict(name='project-test2', result='SUCCESS',
+ changes='2,1 3,1'),
+ ], ordered=False)
+
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
@@ -1787,7 +1878,6 @@
self.assertEqual(self.history[3].result, 'SUCCESS')
self.assertEqual(self.history[3].changes, '1,1 2,2')
- @skip("Disabled for early v3 development")
def test_abandoned_gate(self):
"Test that an abandoned change is dequeued from gate"
@@ -1806,10 +1896,10 @@
self.launch_server.release('.*-merge')
self.waitUntilSettled()
- self.assertEqual(len(self.builds), 0, "No job running")
- self.assertEqual(len(self.history), 1, "Only one build in history")
- self.assertEqual(self.history[0].result, 'ABORTED',
- "Build should have been aborted")
+ self.assertBuilds([])
+ self.assertHistory([
+ dict(name='project-merge', result='ABORTED', changes='1,1')],
+ ordered=False)
self.assertEqual(A.reported, 1,
"Abandoned gate change should report only start")
diff --git a/zuul/configloader.py b/zuul/configloader.py
index bc2f7fc..b41dcc1 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -37,6 +37,29 @@
return [item]
+class NodeSetParser(object):
+ @staticmethod
+ def getSchema():
+ node = {vs.Required('name'): str,
+ vs.Required('image'): str,
+ }
+
+ nodeset = {vs.Required('name'): str,
+ vs.Required('nodes'): [node],
+ }
+
+ return vs.Schema(nodeset)
+
+ @staticmethod
+ def fromYaml(layout, conf):
+ NodeSetParser.getSchema()(conf)
+ ns = model.NodeSet(conf['name'])
+ for conf_node in as_list(conf['nodes']):
+ node = model.Node(conf_node['name'], conf_node['image'])
+ ns.addNode(node)
+ return ns
+
+
class JobParser(object):
@staticmethod
def getSchema():
@@ -77,7 +100,7 @@
'files': to_list(str),
'auth': to_list(auth),
'irrelevant-files': to_list(str),
- 'nodes': [node],
+ 'nodes': vs.Any([node], str),
'timeout': int,
'_source_project': model.Project,
}
@@ -100,6 +123,18 @@
job.voting = conf.get('voting', True)
job.hold_following_changes = conf.get('hold-following-changes', False)
job.mutex = conf.get('mutex', None)
+ if 'nodes' in conf:
+ conf_nodes = conf['nodes']
+ if isinstance(conf_nodes, six.string_types):
+ # This references an existing named nodeset in the layout.
+ ns = layout.nodesets[conf_nodes]
+ else:
+ ns = model.NodeSet()
+ for conf_node in conf_nodes:
+ node = model.Node(conf_node['name'], conf_node['image'])
+ ns.addNode(node)
+ job.nodeset = ns
+
tags = conf.get('tags')
if tags:
# Tags are merged via a union rather than a
@@ -156,9 +191,9 @@
continue
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
- project_pipeline.queue_name = conf.get('queue')
+ project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
- layout, conf_pipeline.get('jobs'))
+ layout, conf_pipeline.get('jobs', []))
return project_template
@staticmethod
@@ -571,6 +606,9 @@
scheduler,
config_pipeline))
+ for config_nodeset in data.nodesets:
+ layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
+
for config_job in data.jobs:
layout.addJob(JobParser.fromYaml(layout, config_job))
diff --git a/zuul/launcher/client.py b/zuul/launcher/client.py
index cd6dcfd..96a524d 100644
--- a/zuul/launcher/client.py
+++ b/zuul/launcher/client.py
@@ -297,9 +297,11 @@
def launch(self, job, item, pipeline, dependent_items=[]):
uuid = str(uuid4().hex)
self.log.info(
- "Launch job %s (uuid: %s) for change %s with dependent "
- "changes %s" % (
- job, uuid, item.change,
+ "Launch job %s (uuid: %s) on nodes %s for change %s "
+ "with dependent changes %s" % (
+ job, uuid,
+ item.current_build_set.getJobNodeSet(job.name),
+ item.change,
[x.change for x in dependent_items]))
dependent_items = dependent_items[:]
dependent_items.reverse()
@@ -371,6 +373,10 @@
params['job'] = job.name
params['items'] = merger_items
params['projects'] = []
+ nodes = []
+ for node in item.current_build_set.getJobNodeSet(job.name).getNodes():
+ nodes.append(dict(name=node.name, image=node.image))
+ params['nodes'] = nodes
projects = set()
for item in all_items:
if item.change.project not in projects:
diff --git a/zuul/launcher/server.py b/zuul/launcher/server.py
index 90c362c..9e29d7f 100644
--- a/zuul/launcher/server.py
+++ b/zuul/launcher/server.py
@@ -394,9 +394,14 @@
job.sendWorkComplete()
def getHostList(self, args):
- # TODOv3: This should get the appropriate nodes from nodepool,
- # or in the unit tests, be overriden to return localhost.
- return [('localhost', dict(ansible_connection='local'))]
+ # TODOv3: the localhost addition is temporary so we have
+ # something to exercise ansible.
+ hosts = [('localhost', dict(ansible_connection='local'))]
+ for node in args['nodes']:
+ # TODOv3: the connection should almost certainly not be
+ # local.
+ hosts.append((node['name'], dict(ansible_connection='local')))
+ return hosts
def prepareAnsibleFiles(self, jobdir, args):
with open(jobdir.inventory, 'w') as inventory:
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 0cd1877..70a510e 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -616,9 +616,11 @@
request = event.request
build_set = request.build_set
build_set.jobNodeRequestComplete(request.job.name, request,
- request.nodes)
- self.log.info("Completed node request %s for job %s of item %s" %
- (request, request.job.name, build_set.item))
+ request.nodeset)
+ self.log.info("Completed node request %s for job %s of item %s "
+ "with nodes %s" %
+ (request, request.job, build_set.item,
+ request.nodeset))
def reportItem(self, item):
if not item.reported:
diff --git a/zuul/manager/dependent.py b/zuul/manager/dependent.py
index 686a593..3d006c2 100644
--- a/zuul/manager/dependent.py
+++ b/zuul/manager/dependent.py
@@ -36,51 +36,38 @@
def buildChangeQueues(self):
self.log.debug("Building shared change queues")
- change_queues = []
+ change_queues = {}
+ project_configs = self.pipeline.layout.project_configs
for project in self.pipeline.getProjects():
- change_queue = model.ChangeQueue(
- self.pipeline,
- window=self.pipeline.window,
- window_floor=self.pipeline.window_floor,
- window_increase_type=self.pipeline.window_increase_type,
- window_increase_factor=self.pipeline.window_increase_factor,
- window_decrease_type=self.pipeline.window_decrease_type,
- window_decrease_factor=self.pipeline.window_decrease_factor)
+ project_config = project_configs[project.name]
+ project_pipeline_config = project_config.pipelines[
+ self.pipeline.name]
+ queue_name = project_pipeline_config.queue_name
+ if queue_name and queue_name in change_queues:
+ change_queue = change_queues[queue_name]
+ else:
+ p = self.pipeline
+ change_queue = model.ChangeQueue(
+ p,
+ window=p.window,
+ window_floor=p.window_floor,
+ window_increase_type=p.window_increase_type,
+ window_increase_factor=p.window_increase_factor,
+ window_decrease_type=p.window_decrease_type,
+ window_decrease_factor=p.window_decrease_factor,
+ name=queue_name)
+ if queue_name:
+ # If this is a named queue, keep track of it in
+ # case it is referenced again. Otherwise, it will
+ # have a name automatically generated from its
+ # constituent projects.
+ change_queues[queue_name] = change_queue
+ self.pipeline.addQueue(change_queue)
+ self.log.debug("Created queue: %s" % change_queue)
change_queue.addProject(project)
- change_queues.append(change_queue)
- self.log.debug("Created queue: %s" % change_queue)
-
- # Iterate over all queues trying to combine them, and keep doing
- # so until they can not be combined further.
- last_change_queues = change_queues
- while True:
- new_change_queues = self.combineChangeQueues(last_change_queues)
- if len(last_change_queues) == len(new_change_queues):
- break
- last_change_queues = new_change_queues
-
- self.log.info(" Shared change queues:")
- for queue in new_change_queues:
- self.pipeline.addQueue(queue)
- self.log.info(" %s containing %s" % (
- queue, queue.generated_name))
-
- def combineChangeQueues(self, change_queues):
- self.log.debug("Combining shared queues")
- new_change_queues = []
- for a in change_queues:
- merged_a = False
- for b in new_change_queues:
- if not a.getJobs().isdisjoint(b.getJobs()):
- self.log.debug("Merging queue %s into %s" % (a, b))
- b.mergeChangeQueue(a)
- merged_a = True
- break # this breaks out of 'for b' and continues 'for a'
- if not merged_a:
- self.log.debug("Keeping queue %s" % (a))
- new_change_queues.append(a)
- return new_change_queues
+ self.log.debug("Added project %s to queue: %s" %
+ (project, change_queue))
def getChangeQueue(self, change, existing=None):
if existing:
diff --git a/zuul/model.py b/zuul/model.py
index ce3d1a2..0aa1ad5 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -131,11 +131,6 @@
def setManager(self, manager):
self.manager = manager
- def addProject(self, project):
- job_tree = JobTree(None) # Null job == job tree root
- self.job_trees[project] = job_tree
- return job_tree
-
def getProjects(self):
# cmp is not in python3, applied idiom from
# http://python-future.org/compatible_idioms.html#cmp
@@ -219,11 +214,13 @@
"""
def __init__(self, pipeline, window=0, window_floor=1,
window_increase_type='linear', window_increase_factor=1,
- window_decrease_type='exponential', window_decrease_factor=2):
+ window_decrease_type='exponential', window_decrease_factor=2,
+ name=None):
self.pipeline = pipeline
- self.name = ''
- self.assigned_name = None
- self.generated_name = None
+ if name:
+ self.name = name
+ else:
+ self.name = ''
self.projects = []
self._jobs = set()
self.queue = []
@@ -244,10 +241,8 @@
if project not in self.projects:
self.projects.append(project)
- names = [x.name for x in self.projects]
- names.sort()
- self.generated_name = ', '.join(names)
- self.name = self.assigned_name or self.generated_name
+ if not self.name:
+ self.name = project.name
def enqueueChange(self, change):
item = QueueItem(self, change)
@@ -349,13 +344,71 @@
return '<Project %s>' % (self.name)
+class Node(object):
+ """A single node for use by a job.
+
+ This may represent a request for a node, or an actual node
+ provided by Nodepool.
+ """
+
+ def __init__(self, name, image):
+ self.name = name
+ self.image = image
+
+ def __repr__(self):
+ return '<Node %s:%s>' % (self.name, self.image)
+
+
+class NodeSet(object):
+ """A set of nodes.
+
+ In configuration, NodeSets are attributes of Jobs indicating that
+ a Job requires nodes matching this description.
+
+ They may appear as top-level configuration objects and be named,
+ or they may appears anonymously in in-line job definitions.
+ """
+
+ def __init__(self, name=None):
+ self.name = name or ''
+ self.nodes = OrderedDict()
+
+ def addNode(self, node):
+ if node.name in self.nodes:
+ raise Exception("Duplicate node in %s" % (self,))
+ self.nodes[node.name] = node
+
+ def getNodes(self):
+ return self.nodes.values()
+
+ def __repr__(self):
+ if self.name:
+ name = self.name + ' '
+ else:
+ name = ''
+ return '<NodeSet %s%s>' % (name, self.nodes)
+
+
+class NodeRequest(object):
+ """A request for a set of nodes."""
+
+ def __init__(self, build_set, job, nodeset):
+ self.build_set = build_set
+ self.job = job
+ self.nodeset = nodeset
+ self.id = uuid4().hex
+
+ def __repr__(self):
+ return '<NodeRequest %s>' % (self.nodeset,)
+
+
class Job(object):
"""A Job represents the defintion of actions to perform."""
attributes = dict(
timeout=None,
# variables={},
- nodes=[],
+ nodeset=NodeSet(),
auth={},
workspace=None,
pre_run=None,
@@ -399,7 +452,7 @@
return self.name
def __repr__(self):
- return '<Job %s>' % (self.name,)
+ return '<Job %s branches: %s>' % (self.name, self.branch_matcher)
def inheritFrom(self, other):
"""Copy the inheritable attributes which have been set on the other
@@ -583,7 +636,7 @@
self.unable_to_merge = False
self.failing_reasons = []
self.merge_state = self.NEW
- self.nodes = {} # job -> nodes
+ self.nodesets = {} # job -> nodeset
self.node_requests = {} # job -> reqs
self.files = RepoFiles()
self.layout = None
@@ -625,9 +678,10 @@
keys.sort()
return [self.builds.get(x) for x in keys]
- def getJobNodes(self, job_name):
- # Return None if not provisioned; [] if no nodes required
- return self.nodes.get(job_name)
+ def getJobNodeSet(self, job_name):
+ # Return None if not provisioned; empty NodeSet if no nodes
+ # required
+ return self.nodesets.get(job_name)
def setJobNodeRequest(self, job_name, req):
if job_name in self.node_requests:
@@ -637,10 +691,10 @@
def getJobNodeRequest(self, job_name):
return self.node_requests.get(job_name)
- def jobNodeRequestComplete(self, job_name, req, nodes):
- if job_name in self.nodes:
+ def jobNodeRequestComplete(self, job_name, req, nodeset):
+ if job_name in self.nodesets:
raise Exception("Prior node request for %s" % (job_name))
- self.nodes[job_name] = nodes
+ self.nodesets[job_name] = nodeset
del self.node_requests[job_name]
@@ -794,7 +848,12 @@
result = build.result
else:
# There is no build for the root of this job tree,
- # so we should run it.
+ # so it has not run yet.
+ nodeset = self.current_build_set.getJobNodeSet(job.name)
+ if nodeset is None:
+ # The nodes for this job are not ready, skip
+ # it for now.
+ continue
if mutex.acquire(self, job):
# If this job needs a mutex, either acquire it or make
# sure that we have it before running the job.
@@ -820,15 +879,12 @@
if job:
if not job.changeMatches(self.change):
continue
- nodes = self.current_build_set.getJobNodes(job.name)
- if nodes is None:
+ nodeset = self.current_build_set.getJobNodeSet(job.name)
+ if nodeset is None:
req = self.current_build_set.getJobNodeRequest(job.name)
if req is None:
toreq.append(job)
- # If there is no job, this is a null job tree, and we should
- # run all of its jobs.
- if not job:
- toreq.extend(self._findJobsToRequest(tree.job_trees))
+ toreq.extend(self._findJobsToRequest(tree.job_trees))
return toreq
def findJobsToRequest(self):
@@ -1573,6 +1629,7 @@
self.jobs = []
self.project_templates = []
self.projects = []
+ self.nodesets = []
def copy(self):
r = UnparsedTenantConfig()
@@ -1580,6 +1637,7 @@
r.jobs = copy.deepcopy(self.jobs)
r.project_templates = copy.deepcopy(self.project_templates)
r.projects = copy.deepcopy(self.projects)
+ r.nodesets = copy.deepcopy(self.nodesets)
return r
def extend(self, conf, source_project=None):
@@ -1588,6 +1646,7 @@
self.jobs.extend(conf.jobs)
self.project_templates.extend(conf.project_templates)
self.projects.extend(conf.projects)
+ self.nodesets.extend(conf.nodesets)
return
if not isinstance(conf, list):
@@ -1614,6 +1673,8 @@
self.project_templates.append(value)
elif key == 'pipeline':
self.pipelines.append(value)
+ elif key == 'nodeset':
+ self.nodesets.append(value)
else:
raise Exception("Configuration item `%s` not recognized "
"(when parsing %s)" %
@@ -1636,6 +1697,7 @@
# that override some attribute of the job. These aspects all
# inherit from the reference definition.
self.jobs = {}
+ self.nodesets = {}
def getJob(self, name):
if name in self.jobs:
@@ -1661,6 +1723,11 @@
else:
self.jobs[job.name] = [job]
+ def addNodeSet(self, nodeset):
+ if nodeset.name in self.nodesets:
+ raise Exception("NodeSet %s already defined" % (nodeset.name,))
+ self.nodesets[nodeset.name] = nodeset
+
def addPipeline(self, pipeline):
self.pipelines[pipeline.name] = pipeline
diff --git a/zuul/nodepool.py b/zuul/nodepool.py
index 85a18f1..addeaf3 100644
--- a/zuul/nodepool.py
+++ b/zuul/nodepool.py
@@ -10,21 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-from uuid import uuid4
-
-
-class Node(object):
- def __init__(self, name, image):
- self.name = name
- self.image = image
-
-
-class Request(object):
- def __init__(self, build_set, job, nodes):
- self.build_set = build_set
- self.job = job
- self.nodes = nodes
- self.id = uuid4().hex
+from zuul.model import NodeRequest
class Nodepool(object):
@@ -33,9 +19,7 @@
self.sched = scheduler
def requestNodes(self, build_set, job):
- nodes = job.nodes
- nodes = [Node(node['name'], node['image']) for node in nodes]
- req = Request(build_set, job, nodes)
+ req = NodeRequest(build_set, job, job.nodeset)
self.requests[req.id] = req
self._requestComplete(req.id)
return req