Merge "Add canonical hostname to source object" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index 9bd44f6..1f447da 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -771,12 +771,15 @@
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
- def runPlaybooks(self, args):
+ def doMergeChanges(self, items):
+ # Get a merger in order to update the repos involved in this job.
+ commit = super(RecordingAnsibleJob, self).doMergeChanges(items)
+ if not commit: # merge conflict
+ self.recordResult('MERGER_FAILURE')
+ return commit
+
+ def recordResult(self, result):
build = self.executor_server.job_builds[self.job.unique]
- build.jobdir = self.jobdir
-
- result = super(RecordingAnsibleJob, self).runPlaybooks(args)
-
self.executor_server.lock.acquire()
self.executor_server.build_history.append(
BuildHistory(name=build.name, result=result, changes=build.changes,
@@ -787,6 +790,13 @@
self.executor_server.running_builds.remove(build)
del self.executor_server.job_builds[self.job.unique]
self.executor_server.lock.release()
+
+ def runPlaybooks(self, args):
+ build = self.executor_server.job_builds[self.job.unique]
+ build.jobdir = self.jobdir
+
+ result = super(RecordingAnsibleJob, self).runPlaybooks(args)
+ self.recordResult(result)
return result
def runAnsible(self, cmd, timeout, trusted=False):
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 47c173d..dff18de 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -151,6 +151,15 @@
- project:
name: org/project2
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
gate:
queue: integrated
jobs:
diff --git a/tests/fixtures/config/single-tenant/git/layout-ignore-dependencies/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-ignore-dependencies/zuul.yaml
new file mode 100644
index 0000000..4010372
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-ignore-dependencies/zuul.yaml
@@ -0,0 +1,66 @@
+- pipeline:
+ name: check
+ manager: independent
+ ignore-dependencies: true
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: project1-merge
+
+- job:
+ name: project1-test1
+
+- job:
+ name: project1-test2
+
+- job:
+ name: project2-merge
+
+- job:
+ name: project2-test1
+
+- job:
+ name: project2-test2
+
+- job:
+ name: project1-project2-integration
+ queue-name: integration
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - project1-merge
+ - project1-test1:
+ dependencies:
+ - project1-merge
+ - project1-test2:
+ dependencies:
+ - project1-merge
+ - project1-project2-integration:
+ dependencies:
+ - project1-merge
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - project2-merge
+ - project2-test1:
+ dependencies:
+ - project2-merge
+ - project2-test2:
+ dependencies:
+ - project2-merge
+ - project1-project2-integration:
+ dependencies:
+ - project2-merge
diff --git a/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-merge.yaml b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-test2.yaml b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-testfile.yaml b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-testfile.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/playbooks/project-testfile.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/zuul.yaml
new file mode 100644
index 0000000..a6d6599
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-live-reconfiguration-del-project/zuul.yaml
@@ -0,0 +1,42 @@
+- pipeline:
+ name: check
+ manager: independent
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: project-merge
+ hold-following-changes: true
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- job:
+ name: project-testfile
+
+- project:
+ name: org/project
+ merge-mode: cherry-pick
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies:
+ - project-merge
+ - project-test2:
+ dependencies:
+ - project-merge
+ - project-testfile:
+ dependencies:
+ - project-merge
diff --git a/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-merge.yaml b/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-test2.yaml b/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-rate-limit/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-rate-limit/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-rate-limit/zuul.yaml
new file mode 100644
index 0000000..c4e00f6
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-rate-limit/zuul.yaml
@@ -0,0 +1,47 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ start:
+ gerrit:
+ verified: 0
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ window: 2
+ window-floor: 1
+ window-increase-type: linear
+ window-increase-factor: 1
+ window-decrease-type: exponential
+ window-decrease-factor: 2
+
+- job:
+ name: project-merge
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies:
+ - project-merge
+ - project-test2:
+ dependencies:
+ - project-merge
diff --git a/tests/fixtures/layout-ignore-dependencies.yaml b/tests/fixtures/layout-ignore-dependencies.yaml
deleted file mode 100644
index 5c0257c..0000000
--- a/tests/fixtures/layout-ignore-dependencies.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- ignore-dependencies: true
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
-projects:
- - name: org/project1
- check:
- - project1-merge:
- - project1-test1
- - project1-test2
- - project1-project2-integration
-
- - name: org/project2
- check:
- - project2-merge:
- - project2-test1
- - project2-test2
- - project1-project2-integration
diff --git a/tests/fixtures/layout-live-reconfiguration-del-project.yaml b/tests/fixtures/layout-live-reconfiguration-del-project.yaml
deleted file mode 100644
index 07ffb2e..0000000
--- a/tests/fixtures/layout-live-reconfiguration-del-project.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
-projects:
- - name: org/project
- merge-mode: cherry-pick
- check:
- - project-merge:
- - project-test1
- - project-test2
- - project-testfile
diff --git a/tests/fixtures/layout-rate-limit.yaml b/tests/fixtures/layout-rate-limit.yaml
deleted file mode 100644
index 9f6748c..0000000
--- a/tests/fixtures/layout-rate-limit.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-pipelines:
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- start:
- gerrit:
- verified: 0
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- window: 2
- window-floor: 1
- window-increase-type: linear
- window-increase-factor: 1
- window-decrease-type: exponential
- window-decrease-factor: 2
-
-projects:
- - name: org/project
- gate:
- - project-merge:
- - project-test1
- - project-test2
diff --git a/tests/make_playbooks.py b/tests/make_playbooks.py
index 17acba8..93c37bc 100755
--- a/tests/make_playbooks.py
+++ b/tests/make_playbooks.py
@@ -14,7 +14,7 @@
import os
-import yaml
+from zuul.lib import yamlutil as yaml
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'fixtures')
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index f906095..2167a3b 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -18,11 +18,11 @@
import fixtures
import testtools
-import yaml
from zuul import model
from zuul import configloader
from zuul.lib import encryption
+from zuul.lib import yamlutil as yaml
from tests.base import BaseTestCase, FIXTURE_DIR
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 4b3fbf4..43a8ddf 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -926,18 +926,17 @@
a = source.getChange(event, refresh=True)
self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
- @skip("Disabled for early v3 development")
- def test_build_configuration_conflict(self):
- "Test that merge conflicts are handled"
+ def test_project_merge_conflict(self):
+ "Test that gate merge conflicts are handled properly"
self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange('org/conflict-project',
- 'master', 'A')
- A.addPatchset(['conflict'])
- B = self.fake_gerrit.addFakeChange('org/conflict-project',
- 'master', 'B')
- B.addPatchset(['conflict'])
- C = self.fake_gerrit.addFakeChange('org/conflict-project',
+ A = self.fake_gerrit.addFakeChange('org/project',
+ 'master', 'A',
+ files={'conflict': 'foo'})
+ B = self.fake_gerrit.addFakeChange('org/project',
+ 'master', 'B',
+ files={'conflict': 'bar'})
+ C = self.fake_gerrit.addFakeChange('org/project',
'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
@@ -951,15 +950,13 @@
self.assertEqual(B.reported, 1)
self.assertEqual(C.reported, 1)
- self.gearman_server.release('.*-merge')
+ self.gearman_server.release('project-merge')
self.waitUntilSettled()
- self.gearman_server.release('.*-merge')
+ self.gearman_server.release('project-merge')
self.waitUntilSettled()
- self.gearman_server.release('.*-merge')
+ self.gearman_server.release('project-merge')
self.waitUntilSettled()
- self.assertEqual(len(self.history), 2) # A and C merge jobs
-
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
@@ -970,7 +967,97 @@
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
- self.assertEqual(len(self.history), 6)
+
+ self.assertHistory([
+ dict(name='project-merge', result='SUCCESS', changes='1,1'),
+ dict(name='project-test1', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),
+ dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
+ ], ordered=False)
+
+ def test_delayed_merge_conflict(self):
+ "Test that delayed check merge conflicts are handled properly"
+
+ # Hold jobs in the gearman queue so that we can test whether
+ # the executor returns a merge failure after the scheduler has
+ # successfully merged.
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('org/project',
+ 'master', 'A',
+ files={'conflict': 'foo'})
+ B = self.fake_gerrit.addFakeChange('org/project',
+ 'master', 'B',
+ files={'conflict': 'bar'})
+ C = self.fake_gerrit.addFakeChange('org/project',
+ 'master', 'C')
+ C.setDependsOn(B, 1)
+
+ # A enters the gate queue; B and C enter the check queue
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 0) # Check does not report start
+ self.assertEqual(C.reported, 0) # Check does not report start
+
+ # A merges while B and C are queued in check
+ # Release A project-merge
+ queue = self.gearman_server.getQueue()
+ self.release(queue[0])
+ self.waitUntilSettled()
+
+ # Release A project-test*
+ # gate has higher precedence, so A's test jobs are added in
+ # front of the merge jobs for B and C
+ queue = self.gearman_server.getQueue()
+ self.release(queue[0])
+ self.release(queue[1])
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(C.data['status'], 'NEW')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(C.reported, 0)
+ self.assertHistory([
+ dict(name='project-merge', result='SUCCESS', changes='1,1'),
+ dict(name='project-test1', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+ # B and C report merge conflicts
+ # Release B project-merge
+ queue = self.gearman_server.getQueue()
+ self.release(queue[0])
+ self.waitUntilSettled()
+
+ # Release C
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(C.data['status'], 'NEW')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 1)
+ self.assertEqual(C.reported, 1)
+
+ self.assertHistory([
+ dict(name='project-merge', result='SUCCESS', changes='1,1'),
+ dict(name='project-test1', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ dict(name='project-merge', result='MERGER_FAILURE', changes='2,1'),
+ dict(name='project-merge', result='MERGER_FAILURE',
+ changes='2,1 3,1'),
+ ], ordered=False)
def test_post(self):
"Test that post jobs run"
@@ -2719,7 +2806,6 @@
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
- @skip("Disabled for early v3 development")
def test_live_reconfiguration_del_project(self):
# Test project deletion from layout
# while changes are enqueued
@@ -2742,14 +2828,14 @@
self.assertEqual(len(self.builds), 5)
# This layout defines only org/project, not org/project1
- self.updateConfigLayout(
- 'tests/fixtures/layout-live-reconfiguration-del-project.yaml')
+ self.commitLayoutUpdate('common-config',
+ 'layout-live-reconfiguration-del-project')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
# Builds for C aborted, builds for A succeed,
# and have change B applied ahead
- job_c = self.getJobFromHistory('project1-test1')
+ job_c = self.getJobFromHistory('project-test1')
self.assertEqual(job_c.changes, '3,1')
self.assertEqual(job_c.result, 'ABORTED')
@@ -2757,8 +2843,9 @@
self.executor_server.release()
self.waitUntilSettled()
- self.assertEqual(self.getJobFromHistory('project-test1').changes,
- '2,1 1,1')
+ self.assertEqual(
+ self.getJobFromHistory('project-test1', 'org/project').changes,
+ '2,1 1,1')
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
@@ -2767,40 +2854,11 @@
self.assertEqual(B.reported, 0)
self.assertEqual(C.reported, 0)
- self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0)
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
self.assertIn('Build succeeded', A.messages[0])
@skip("Disabled for early v3 development")
- def test_live_reconfiguration_functions(self):
- "Test live reconfiguration with a custom function"
- self.worker.registerFunction('build:node-project-test1:debian')
- self.worker.registerFunction('build:node-project-test1:wheezy')
- A = self.fake_gerrit.addFakeChange('org/node-project', 'master', 'A')
- A.addApproval('code-review', 2)
- self.fake_gerrit.addEvent(A.addApproval('approved', 1))
- self.waitUntilSettled()
-
- self.assertIsNone(self.getJobFromHistory('node-project-merge').node)
- self.assertEqual(self.getJobFromHistory('node-project-test1').node,
- 'debian')
- self.assertIsNone(self.getJobFromHistory('node-project-test2').node)
-
- self.updateConfigLayout(
- 'tests/fixtures/layout-live-reconfiguration-functions.yaml')
- self.sched.reconfigure(self.config)
- self.worker.build_history = []
-
- B = self.fake_gerrit.addFakeChange('org/node-project', 'master', 'B')
- B.addApproval('code-review', 2)
- self.fake_gerrit.addEvent(B.addApproval('approved', 1))
- self.waitUntilSettled()
-
- self.assertIsNone(self.getJobFromHistory('node-project-merge').node)
- self.assertEqual(self.getJobFromHistory('node-project-test1').node,
- 'wheezy')
- self.assertIsNone(self.getJobFromHistory('node-project-test2').node)
-
- @skip("Disabled for early v3 development")
def test_delayed_repo_init(self):
self.updateConfigLayout(
'tests/fixtures/layout-delayed-repo-init.yaml')
@@ -3349,11 +3407,9 @@
self.executor_server.release()
self.waitUntilSettled()
- @skip("Disabled for early v3 development")
def test_queue_rate_limiting(self):
"Test that DependentPipelines are rate limited with dep across window"
- self.updateConfigLayout(
- 'tests/fixtures/layout-rate-limit.yaml')
+ self.updateConfigLayout('layout-rate-limit')
self.sched.reconfigure(self.config)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
@@ -3394,7 +3450,8 @@
self.executor_server.release('project-.*')
self.waitUntilSettled()
- queue = self.sched.layout.pipelines['gate'].queues[0]
+ tenant = self.sched.abide.tenants.get('openstack')
+ queue = tenant.layout.pipelines['gate'].queues[0]
# A failed so window is reduced by 1 to 1.
self.assertEqual(queue.window, 1)
self.assertEqual(queue.window_floor, 1)
@@ -3441,11 +3498,9 @@
self.assertEqual(queue.window_floor, 1)
self.assertEqual(C.data['status'], 'MERGED')
- @skip("Disabled for early v3 development")
def test_queue_rate_limiting_dependent(self):
"Test that DependentPipelines are rate limited with dep in window"
- self.updateConfigLayout(
- 'tests/fixtures/layout-rate-limit.yaml')
+ self.updateConfigLayout('layout-rate-limit')
self.sched.reconfigure(self.config)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
@@ -3487,7 +3542,8 @@
self.executor_server.release('project-.*')
self.waitUntilSettled()
- queue = self.sched.layout.pipelines['gate'].queues[0]
+ tenant = self.sched.abide.tenants.get('openstack')
+ queue = tenant.layout.pipelines['gate'].queues[0]
# A failed so window is reduced by 1 to 1.
self.assertEqual(queue.window, 1)
self.assertEqual(queue.window_floor, 1)
@@ -4233,13 +4289,10 @@
self.init_repo("org/unknown")
self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
- @skip("Disabled for early v3 development")
def test_crd_check_ignore_dependencies(self):
"Test cross-repo dependencies can be ignored"
- self.updateConfigLayout(
- 'tests/fixtures/layout-ignore-dependencies.yaml')
+ self.updateConfigLayout('layout-ignore-dependencies')
self.sched.reconfigure(self.config)
- self.registerJobs()
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
@@ -4258,7 +4311,8 @@
# Make sure none of the items share a change queue, and all
# are live.
- check_pipeline = self.sched.layout.pipelines['check']
+ tenant = self.sched.abide.tenants.get('openstack')
+ check_pipeline = tenant.layout.pipelines['check']
self.assertEqual(len(check_pipeline.queues), 3)
self.assertEqual(len(check_pipeline.getAllItems()), 3)
for item in check_pipeline.getAllItems():
@@ -4279,7 +4333,6 @@
for job in self.history:
self.assertEqual(len(job.changes.split()), 1)
- @skip("Disabled for early v3 development")
def test_crd_check_transitive(self):
"Test transitive cross-repo dependencies"
# Specifically, if A -> B -> C, and C gets a new patchset and
diff --git a/tools/test-setup.sh b/tools/test-setup.sh
index f4a0458..3bdedf5 100755
--- a/tools/test-setup.sh
+++ b/tools/test-setup.sh
@@ -6,6 +6,10 @@
# This setup needs to be run as a user that can run sudo.
+# Be sure mysql and zookeeper are started.
+sudo service mysql start
+sudo service zookeeper start
+
# The root password for the MySQL database; pass it in via
# MYSQL_ROOT_PW.
DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave}
diff --git a/zuul/ansible/lookup/__init__.py b/zuul/ansible/lookup/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/zuul/ansible/lookup/__init__.py
diff --git a/zuul/ansible/lookup/_banned.py b/zuul/ansible/lookup/_banned.py
new file mode 100644
index 0000000..65708f8
--- /dev/null
+++ b/zuul/ansible/lookup/_banned.py
@@ -0,0 +1,25 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+
+
+class LookupModule(LookupBase):
+
+ def run(self, *args, **kwargs):
+ raise AnsibleError(
+ "Use of lookup modules that perform local actions on the executor"
+ " is forbidden.")
diff --git a/zuul/ansible/lookup/consul_kv.py b/zuul/ansible/lookup/consul_kv.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/consul_kv.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/credstash.py b/zuul/ansible/lookup/credstash.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/credstash.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/csvfile.py b/zuul/ansible/lookup/csvfile.py
new file mode 100644
index 0000000..6506aa2
--- /dev/null
+++ b/zuul/ansible/lookup/csvfile.py
@@ -0,0 +1,25 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+csvfile = paths._import_ansible_lookup_plugin("csvfile")
+
+
+class LookupModule(csvfile.LookupModule):
+
+ def read_csv(self, filename, *args, **kwargs):
+ paths._fail_if_unsafe(filename)
+ return super(LookupModule, self).read_csv(filename, *args, **kwargs)
diff --git a/zuul/ansible/lookup/dig.py b/zuul/ansible/lookup/dig.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/dig.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/dnstxt.py b/zuul/ansible/lookup/dnstxt.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/dnstxt.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/env.py b/zuul/ansible/lookup/env.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/env.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/etcd.py b/zuul/ansible/lookup/etcd.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/etcd.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/file.py b/zuul/ansible/lookup/file.py
new file mode 100644
index 0000000..7403535
--- /dev/null
+++ b/zuul/ansible/lookup/file.py
@@ -0,0 +1,28 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+file_mod = paths._import_ansible_lookup_plugin("file")
+
+
+class LookupModule(file_mod.LookupModule):
+
+ def run(self, terms, variables=None, **kwargs):
+ for term in terms:
+ lookupfile = self.find_file_in_search_path(
+ variables, 'files', term)
+ paths._fail_if_unsafe(lookupfile)
+ return super(LookupModule, self).run(terms, variables, **kwargs)
diff --git a/zuul/ansible/lookup/fileglob.py b/zuul/ansible/lookup/fileglob.py
new file mode 100644
index 0000000..4b9b449
--- /dev/null
+++ b/zuul/ansible/lookup/fileglob.py
@@ -0,0 +1,45 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+# Forked from lib/ansible/plugins/lookup/fileglob.py in ansible
+
+import os
+import glob
+
+from zuul.ansible import paths
+
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils._text import to_bytes, to_text
+
+
+class LookupModule(LookupBase):
+
+ def run(self, terms, variables=None, **kwargs):
+
+ ret = []
+ for term in terms:
+ term_file = os.path.basename(term)
+ dwimmed_path = self.find_file_in_search_path(
+ variables, 'files', os.path.dirname(term))
+ if dwimmed_path:
+ paths._fail_if_unsafe(dwimmed_path)
+ globbed = glob.glob(to_bytes(
+ os.path.join(dwimmed_path, term_file),
+ errors='surrogate_or_strict'))
+ ret.extend(
+ to_text(g, errors='surrogate_or_strict')
+ for g in globbed if os.path.isfile(g))
+ return ret
diff --git a/zuul/ansible/lookup/filetree.py b/zuul/ansible/lookup/filetree.py
new file mode 100644
index 0000000..0c054a3
--- /dev/null
+++ b/zuul/ansible/lookup/filetree.py
@@ -0,0 +1,32 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from zuul.ansible import paths
+filetree = paths._import_ansible_lookup_plugin("filetree")
+
+
+class LookupModule(filetree.LookupModule):
+
+ def run(self, terms, variables=None, **kwargs):
+ basedir = self.get_basedir(variables)
+ for term in terms:
+ term_file = os.path.basename(term)
+ dwimmed_path = self._loader.path_dwim_relative(
+ basedir, 'files', os.path.dirname(term))
+ path = os.path.join(dwimmed_path, term_file)
+ paths._fail_if_unsafe(path)
+ return super(LookupModule, self).run(terms, variables, **kwargs)
diff --git a/zuul/ansible/lookup/first_found.py b/zuul/ansible/lookup/first_found.py
new file mode 100644
index 0000000..d741df0
--- /dev/null
+++ b/zuul/ansible/lookup/first_found.py
@@ -0,0 +1,201 @@
+# (c) 2013, seth vidal <skvidal@fedoraproject.org> red hat, inc
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+# take a list of files and (optionally) a list of paths
+# return the first existing file found in the paths
+# [file1, file2, file3], [path1, path2, path3]
+# search order is:
+# path1/file1
+# path1/file2
+# path1/file3
+# path2/file1
+# path2/file2
+# path2/file3
+# path3/file1
+# path3/file2
+# path3/file3
+
+# first file found with os.path.exists() is returned
+# no file matches raises ansibleerror
+# EXAMPLES
+# - name: copy first existing file found to /some/file
+# action: copy src=$item dest=/some/file
+# with_first_found:
+# - files: foo ${inventory_hostname} bar
+# paths: /tmp/production /tmp/staging
+
+# that will look for files in this order:
+# /tmp/production/foo
+# ${inventory_hostname}
+# bar
+# /tmp/staging/foo
+# ${inventory_hostname}
+# bar
+
+# - name: copy first existing file found to /some/file
+# action: copy src=$item dest=/some/file
+# with_first_found:
+# - files: /some/place/foo ${inventory_hostname} /some/place/else
+
+# that will look for files in this order:
+# /some/place/foo
+# $relative_path/${inventory_hostname}
+# /some/place/else
+
+# example - including tasks:
+# tasks:
+# - include: $item
+# with_first_found:
+# - files: generic
+# paths: tasks/staging tasks/production
+# this will include the tasks in the file generic where it is found first
+# (staging or production)
+
+# example simple file lists
+# tasks:
+# - name: first found file
+# action: copy src=$item dest=/etc/file.cfg
+# with_first_found:
+# - files: foo.${inventory_hostname} foo
+
+
+# example skipping if no matched files
+# First_found also offers the ability to control whether or not failing
+# to find a file returns an error or not
+#
+# - name: first found file - or skip
+# action: copy src=$item dest=/etc/file.cfg
+# with_first_found:
+# - files: foo.${inventory_hostname}
+# skip: true
+
+# example a role with default configuration and configuration per host
+# you can set multiple terms with their own files and paths to look through.
+# consider a role that sets some configuration per host falling back on a
+# default config.
+#
+# - name: some configuration template
+# template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root
+# with_first_found:
+# - files:
+# - ${inventory_hostname}/etc/file.cfg
+# paths:
+# - ../../../templates.overwrites
+# - ../../../templates
+# - files:
+# - etc/file.cfg
+# paths:
+# - templates
+
+# the above will return an empty list if the files cannot be found at all
+# if skip is unspecificed or if it is set to false then it will return a list
+# error which can be caught bye ignore_errors: true for that action.
+
+# finally - if you want you can use it, in place to replace
+# first_available_file:
+# you simply cannot use the - files, path or skip options. simply replace
+# first_available_file with with_first_found and leave the file listing in
+# place
+#
+#
+# - name: with_first_found like first_available_file
+# action: copy src=$item dest=/tmp/faftest
+# with_first_found:
+# - ../files/foo
+# - ../files/bar
+# - ../files/baz
+# ignore_errors: true
+
+import os
+
+from jinja2.exceptions import UndefinedError
+
+from ansible.constants import mk_boolean as boolean
+from ansible.errors import AnsibleLookupError
+from ansible.errors import AnsibleUndefinedVariable
+from ansible.module_utils.six import string_types
+from ansible.plugins.lookup import LookupBase
+
+from zuul.ansible import paths as zuul_paths
+
+
+class LookupModule(LookupBase):
+
+ def run(self, terms, variables, **kwargs):
+
+ anydict = False
+ skip = False
+
+ for term in terms:
+ if isinstance(term, dict):
+ anydict = True
+
+ total_search = []
+ if anydict:
+ for term in terms:
+ if isinstance(term, dict):
+ files = term.get('files', [])
+ paths = term.get('paths', [])
+ skip = boolean(term.get('skip', False))
+
+ filelist = files
+ if isinstance(files, string_types):
+ files = files.replace(',', ' ')
+ files = files.replace(';', ' ')
+ filelist = files.split(' ')
+
+ pathlist = paths
+ if paths:
+ if isinstance(paths, string_types):
+ paths = paths.replace(',', ' ')
+ paths = paths.replace(':', ' ')
+ paths = paths.replace(';', ' ')
+ pathlist = paths.split(' ')
+
+ if not pathlist:
+ total_search = filelist
+ else:
+ for path in pathlist:
+ for fn in filelist:
+ f = os.path.join(path, fn)
+ total_search.append(f)
+ else:
+ total_search.append(term)
+ else:
+ total_search = self._flatten(terms)
+
+ for fn in total_search:
+ zuul_paths._fail_if_unsafe(fn)
+ try:
+ fn = self._templar.template(fn)
+ except (AnsibleUndefinedVariable, UndefinedError):
+ continue
+
+ # get subdir if set by task executor, default to files otherwise
+ subdir = getattr(self, '_subdir', 'files')
+ path = None
+ path = self.find_file_in_search_path(
+ variables, subdir, fn, ignore_missing=True)
+ if path is not None:
+ return [path]
+ else:
+ if skip:
+ return []
+ else:
+ raise AnsibleLookupError(
+ "No file was found when using with_first_found. Use the"
+ " 'skip: true' option to allow this task to be skipped if"
+ " no files are found")
diff --git a/zuul/ansible/lookup/hashi_valut.py b/zuul/ansible/lookup/hashi_valut.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/hashi_valut.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/ini.py b/zuul/ansible/lookup/ini.py
new file mode 100644
index 0000000..51127ff
--- /dev/null
+++ b/zuul/ansible/lookup/ini.py
@@ -0,0 +1,31 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+ini = paths._import_ansible_lookup_plugin("ini")
+
+
+class LookupModule(ini.LookupModule):
+
+ def read_properties(self, filename, *args, **kwargs):
+ paths._fail_if_unsafe(filename)
+ return super(LookupModule, self).read_properties(
+ filename, *args, **kwargs)
+
+ def read_ini(self, filename, *args, **kwargs):
+ paths._fail_if_unsafe(filename)
+ return super(LookupModule, self).read_ini(
+ filename, *args, **kwargs)
diff --git a/zuul/ansible/lookup/keyring.py b/zuul/ansible/lookup/keyring.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/keyring.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/lastpass.py b/zuul/ansible/lookup/lastpass.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/lastpass.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/lines.py b/zuul/ansible/lookup/lines.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/lines.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/mongodb.py b/zuul/ansible/lookup/mongodb.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/mongodb.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/password.py b/zuul/ansible/lookup/password.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/password.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/passwordstore.py b/zuul/ansible/lookup/passwordstore.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/passwordstore.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/pipe.py b/zuul/ansible/lookup/pipe.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/pipe.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/redis_kv.py b/zuul/ansible/lookup/redis_kv.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/redis_kv.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/shelvefile.py b/zuul/ansible/lookup/shelvefile.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/shelvefile.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/template.py b/zuul/ansible/lookup/template.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/template.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/url.py b/zuul/ansible/lookup/url.py
new file mode 120000
index 0000000..d45b9c4
--- /dev/null
+++ b/zuul/ansible/lookup/url.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/paths.py b/zuul/ansible/paths.py
index e387732..bc61975 100644
--- a/zuul/ansible/paths.py
+++ b/zuul/ansible/paths.py
@@ -16,7 +16,9 @@
import imp
import os
+from ansible.errors import AnsibleError
import ansible.plugins.action
+import ansible.plugins.lookup
def _is_safe_path(path):
@@ -35,6 +37,12 @@
curdir=os.path.abspath(os.path.curdir)))
+def _fail_if_unsafe(path):
+ if not _is_safe_path(path):
+ msg_dict = _fail_dict(path)
+ raise AnsibleError(msg_dict['msg'])
+
+
def _import_ansible_action_plugin(name):
# Ansible forces the import of our action plugins
# (zuul.ansible.action.foo) as ansible.plugins.action.foo, which
@@ -51,3 +59,11 @@
return imp.load_module(
'zuul.ansible.protected.action.' + name,
*imp.find_module(name, ansible.plugins.action.__path__))
+
+
+def _import_ansible_lookup_plugin(name):
+ # See _import_ansible_action_plugin
+
+ return imp.load_module(
+ 'zuul.ansible.protected.lookup.' + name,
+ *imp.find_module(name, ansible.plugins.lookup.__path__))
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index 9fa4c03..f2a2612 100644
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -24,10 +24,10 @@
import sys
import traceback
-import yaml
yappi = extras.try_import('yappi')
import zuul.lib.connections
+from zuul.lib import yamlutil as yaml
# Do not import modules that will pull in paramiko which must not be
# imported until after the daemonization.
diff --git a/zuul/configloader.py b/zuul/configloader.py
index ecba760..5e88ee7 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -15,13 +15,13 @@
import os
import logging
import six
-import yaml
import pprint
import textwrap
import voluptuous as vs
from zuul import model
+from zuul.lib import yamlutil as yaml
import zuul.manager.dependent
import zuul.manager.independent
from zuul import change_matcher
diff --git a/zuul/executor/ansiblelaunchserver.py b/zuul/executor/ansiblelaunchserver.py
index 875cf2b..0202bdd 100644
--- a/zuul/executor/ansiblelaunchserver.py
+++ b/zuul/executor/ansiblelaunchserver.py
@@ -35,13 +35,13 @@
import Queue
import gear
-import yaml
import jenkins_jobs.builder
import jenkins_jobs.formatter
import zmq
import zuul.ansible.library
from zuul.lib import commandsocket
+from zuul.lib import yamlutil as yaml
ANSIBLE_WATCHDOG_GRACE = 5 * 60
ANSIBLE_DEFAULT_TIMEOUT = 2 * 60 * 60
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 027c63c..0adb6de 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -24,15 +24,17 @@
import threading
import time
import traceback
-import yaml
+from zuul.lib.yamlutil import yaml
import gear
import git
+from six.moves import shlex_quote
import zuul.merger.merger
import zuul.ansible.action
import zuul.ansible.callback
import zuul.ansible.library
+import zuul.ansible.lookup
from zuul.lib import commandsocket
COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
@@ -274,6 +276,10 @@
if not os.path.exists(self.callback_dir):
os.makedirs(self.callback_dir)
+ self.lookup_dir = os.path.join(ansible_dir, 'lookup')
+ if not os.path.exists(self.lookup_dir):
+ os.makedirs(self.lookup_dir)
+
library_path = os.path.dirname(os.path.abspath(
zuul.ansible.library.__file__))
for fn in os.listdir(library_path):
@@ -289,6 +295,11 @@
for fn in os.listdir(callback_path):
shutil.copy(os.path.join(callback_path, fn), self.callback_dir)
+ lookup_path = os.path.dirname(os.path.abspath(
+ zuul.ansible.lookup.__file__))
+ for fn in os.listdir(lookup_path):
+ shutil.copy(os.path.join(lookup_path, fn), self.lookup_dir)
+
self.job_workers = {}
def _getMerger(self, root):
@@ -467,7 +478,10 @@
result = dict(merged=(ret is not None),
zuul_url=self.zuul_url)
if args.get('files'):
- result['commit'], result['files'] = ret
+ if ret:
+ result['commit'], result['files'] = ret
+ else:
+ result['commit'], result['files'] = (None, None)
else:
result['commit'] = ret
job.sendWorkComplete(json.dumps(result))
@@ -552,11 +566,13 @@
project['name']))
repo.remotes.origin.config_writer.set('url', project['url'])
- # Get a merger in order to update the repos involved in this job.
- merger = self.executor_server._getMerger(self.jobdir.src_root)
merge_items = [i for i in args['items'] if i.get('refspec')]
if merge_items:
- commit = merger.mergeChanges(merge_items) # noqa
+ commit = self.doMergeChanges(merge_items)
+ if not commit:
+ # There was a merge conflict and we have already sent
+ # a work complete result, don't run any jobs
+ return
else:
commit = args['items'][-1]['newrev'] # noqa
@@ -596,6 +612,15 @@
result = dict(result=result)
self.job.sendWorkComplete(json.dumps(result))
+ def doMergeChanges(self, items):
+ # Get a merger in order to update the repos involved in this job.
+ merger = self.executor_server._getMerger(self.jobdir.src_root)
+ commit = merger.mergeChanges(items) # noqa
+ if not commit: # merge conflict
+ result = dict(result='MERGER_FAILURE')
+ self.job.sendWorkComplete(json.dumps(result))
+ return commit
+
def runPlaybooks(self, args):
result = None
@@ -871,6 +896,8 @@
if not trusted:
config.write('action_plugins = %s\n'
% self.executor_server.action_dir)
+ config.write('lookup_plugins = %s\n'
+ % self.executor_server.lookup_dir)
# On trusted jobs, we want to prevent the printing of args,
# since trusted jobs might have access to secrets that they may
@@ -916,14 +943,17 @@
env_copy['LOGNAME'] = 'zuul'
if trusted:
- env_copy['ANSIBLE_CONFIG'] = self.jobdir.trusted_config
+ config_file = self.jobdir.trusted_config
else:
- env_copy['ANSIBLE_CONFIG'] = self.jobdir.untrusted_config
+ config_file = self.jobdir.untrusted_config
+
+ env_copy['ANSIBLE_CONFIG'] = config_file
with self.proc_lock:
if self.aborted:
return (self.RESULT_ABORTED, None)
- self.log.debug("Ansible command: %s" % (cmd,))
+ self.log.debug("Ansible command: ANSIBLE_CONFIG=%s %s",
+ config_file, " ".join(shlex_quote(c) for c in cmd))
self.proc = subprocess.Popen(
cmd,
cwd=self.jobdir.work_root,
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 18dea91..bec8ebe 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -17,13 +17,13 @@
import logging
import os
import re
-import yaml
import six
from git import GitCommandError
from zuul import exceptions
from zuul.lib.clonemapper import CloneMapper
+from zuul.lib import yamlutil as yaml
from zuul.merger.merger import Repo
diff --git a/zuul/lib/yamlutil.py b/zuul/lib/yamlutil.py
new file mode 100644
index 0000000..2419906
--- /dev/null
+++ b/zuul/lib/yamlutil.py
@@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import yaml
+from yaml import YAMLObject, YAMLError # noqa: F401
+
+try:
+ from yaml import cyaml
+ import _yaml
+ SafeLoader = cyaml.CSafeLoader
+ SafeDumper = cyaml.CSafeDumper
+ Mark = _yaml.Mark
+except ImportError:
+ SafeLoader = yaml.SafeLoader
+ SafeDumper = yaml.SafeDumper
+ Mark = yaml.Mark
+
+
+def safe_load(stream, *args, **kwargs):
+ return yaml.load(stream, *args, Loader=SafeLoader, **kwargs)
+
+
+def safe_dump(stream, *args, **kwargs):
+ return yaml.dump(stream, *args, Dumper=SafeDumper, **kwargs)
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 75e8edb..9507d15 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -442,11 +442,15 @@
# the merger.
number = None
patchset = None
+ refspec = None
+ branch = None
oldrev = None
newrev = None
if hasattr(item.change, 'number'):
number = item.change.number
patchset = item.change.patchset
+ refspec = item.change.refspec
+ branch = item.change.branch
elif hasattr(item.change, 'newrev'):
oldrev = item.change.oldrev
newrev = item.change.newrev
@@ -458,8 +462,8 @@
item.change.project),
connection_name=connection_name,
merge_mode=item.current_build_set.getMergeMode(project),
- refspec=item.change.refspec,
- branch=item.change.branch,
+ refspec=refspec,
+ branch=branch,
ref=item.current_build_set.ref,
number=number,
patchset=patchset,
@@ -517,30 +521,54 @@
if build_set.merge_state == build_set.COMPLETE:
if build_set.unable_to_merge:
return None
+ self.log.debug("Preparing dynamic layout for: %s" % item.change)
return self._loadDynamicLayout(item)
- build_set.merge_state = build_set.PENDING
- self.log.debug("Preparing dynamic layout for: %s" % item.change)
+
+ def scheduleMerge(self, item, files=None):
+ build_set = item.current_build_set
+
+ if not hasattr(item.change, 'branch'):
+ self.log.debug("Change %s does not have an associated branch, "
+ "not scheduling a merge job for item %s" %
+ (item.change, item))
+ build_set.merge_state = build_set.COMPLETE
+ return True
+
+ self.log.debug("Scheduling merge for item %s (files: %s)" %
+ (item, files))
dependent_items = self.getDependentItems(item)
dependent_items.reverse()
all_items = dependent_items + [item]
merger_items = map(self._makeMergerItem, all_items)
+ build_set = item.current_build_set
+ build_set.merge_state = build_set.PENDING
self.sched.merger.mergeChanges(merger_items,
item.current_build_set,
- ['zuul.yaml', '.zuul.yaml'],
+ files,
self.pipeline.precedence)
+ return False
- def prepareLayout(self, item):
- # Get a copy of the layout in the context of the current
- # queue.
- # Returns True if the ref is ready, false otherwise
- if not item.current_build_set.ref:
- item.current_build_set.setConfiguration()
- if not item.current_build_set.layout:
- item.current_build_set.layout = self.getLayout(item)
- if not item.current_build_set.layout:
+ def prepareItem(self, item):
+ # This runs on every iteration of _processOneItem
+ # Returns True if the item is ready, false otherwise
+ build_set = item.current_build_set
+ if not build_set.ref:
+ build_set.setConfiguration()
+ if build_set.merge_state == build_set.NEW:
+ return self.scheduleMerge(item, ['zuul.yaml', '.zuul.yaml'])
+ if build_set.config_error:
return False
- if item.current_build_set.config_error:
+ return True
+
+ def prepareJobs(self, item):
+ # This only runs once the item is in the pipeline's action window
+ # Returns True if the item is ready, false otherwise
+ build_set = item.current_build_set
+ if not build_set.layout:
+ build_set.layout = self.getLayout(item)
+ if not build_set.layout:
return False
+
if not item.job_graph:
try:
item.freezeJobGraph()
@@ -555,11 +583,13 @@
def _processOneItem(self, item, nnfi):
changed = False
+ ready = False
+ failing_reasons = [] # Reasons this item is failing
+
item_ahead = item.item_ahead
if item_ahead and (not item_ahead.live):
item_ahead = None
change_queue = item.queue
- failing_reasons = [] # Reasons this item is failing
if self.checkForChangesNeededBy(item.change, change_queue) is not True:
# It's not okay to enqueue this change, we should remove it.
@@ -574,10 +604,11 @@
except exceptions.MergeFailure:
pass
return (True, nnfi)
- dep_items = self.getFailingDependentItems(item)
+
actionable = change_queue.isActionable(item)
item.active = actionable
- ready = False
+
+ dep_items = self.getFailingDependentItems(item)
if dep_items:
failing_reasons.append('a needed change is failing')
self.cancelJobs(item, prime=False)
@@ -596,15 +627,16 @@
changed = True
self.cancelJobs(item)
if actionable:
- ready = self.prepareLayout(item)
+ ready = self.prepareItem(item) and self.prepareJobs(item)
if item.current_build_set.unable_to_merge:
failing_reasons.append("it has a merge conflict")
if item.current_build_set.config_error:
failing_reasons.append("it has an invalid configuration")
if ready and self.provisionNodes(item):
changed = True
- if actionable and ready and self.executeJobs(item):
+ if ready and self.executeJobs(item):
changed = True
+
if item.didAnyJobFail():
failing_reasons.append("at least one job failed")
if (not item.live) and (not item.items_behind):
@@ -742,10 +774,11 @@
# TODOv3(jeblair): consider a new reporter action for this
actions = self.pipeline.merge_failure_actions
item.setReportedResult('CONFIG_ERROR')
+ elif item.didMergerFail():
+ actions = self.pipeline.merge_failure_actions
+ item.setReportedResult('MERGER_FAILURE')
elif not item.getJobs():
- # We don't send empty reports with +1,
- # and the same for -1's (merge failures or transient errors)
- # as they cannot be followed by +1's
+ # We don't send empty reports with +1
self.log.debug("No jobs for change %s" % item.change)
actions = []
elif item.didAllJobsSucceed():
@@ -753,9 +786,6 @@
actions = self.pipeline.success_actions
item.setReportedResult('SUCCESS')
self.pipeline._consecutive_failures = 0
- elif item.didMergerFail():
- actions = self.pipeline.merge_failure_actions
- item.setReportedResult('MERGER_FAILURE')
else:
actions = self.pipeline.failure_actions
item.setReportedResult('FAILURE')
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index c2738a2..540105e 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -111,7 +111,10 @@
result = dict(merged=(ret is not None),
zuul_url=self.zuul_url)
if args.get('files'):
- result['commit'], result['files'] = ret
+ if ret:
+ result['commit'], result['files'] = ret
+ else:
+ result['commit'], result['files'] = (None, None)
else:
result['commit'] = ret
job.sendWorkComplete(json.dumps(result))