Merge "Add __repr__ to jobtree" into feature/zuulv3
diff --git a/setup.cfg b/setup.cfg
index 7ddeb84..bd76d8b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -21,7 +21,7 @@
[entry_points]
console_scripts =
- zuul-server = zuul.cmd.server:main
+ zuul-scheduler = zuul.cmd.scheduler:main
zuul-merger = zuul.cmd.merger:main
zuul = zuul.cmd.client:main
zuul-cloner = zuul.cmd.cloner:main
diff --git a/tests/base.py b/tests/base.py
index f631ec3..41fa29f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -474,9 +474,6 @@
def getGitUrl(self, project):
return os.path.join(self.upstream_root, project.name)
- def _getGitwebUrl(self, project, sha=None):
- return self.getGitwebUrl(project, sha)
-
class BuildHistory(object):
def __init__(self, **kw):
@@ -1195,7 +1192,8 @@
tmp_root = os.environ.get("ZUUL_TEST_ROOT")
self.test_root = os.path.join(tmp_root, "zuul-test")
self.upstream_root = os.path.join(self.test_root, "upstream")
- self.git_root = os.path.join(self.test_root, "git")
+ self.merger_git_root = os.path.join(self.test_root, "merger-git")
+ self.launcher_git_root = os.path.join(self.test_root, "launcher-git")
self.state_root = os.path.join(self.test_root, "lib")
if os.path.exists(self.test_root):
@@ -1209,7 +1207,8 @@
self.config.set('zuul', 'tenant_config',
os.path.join(FIXTURE_DIR,
self.config.get('zuul', 'tenant_config')))
- self.config.set('merger', 'git_dir', self.git_root)
+ self.config.set('merger', 'git_dir', self.merger_git_root)
+ self.config.set('launcher', 'git_dir', self.launcher_git_root)
self.config.set('zuul', 'state_dir', self.state_root)
# For each project in config:
diff --git a/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
new file mode 100644
index 0000000..0e332e4
--- /dev/null
+++ b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
@@ -0,0 +1,22 @@
+- pipeline:
+ name: check
+ manager: independent
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: project-test1
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/config/git-driver/git/org_project/README b/tests/fixtures/config/git-driver/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/git-driver/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/git-driver/main.yaml b/tests/fixtures/config/git-driver/main.yaml
new file mode 100644
index 0000000..5b9b3d9
--- /dev/null
+++ b/tests/fixtures/config/git-driver/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ git:
+ config-repos:
+ - common-config
+ gerrit:
+ project-repos:
+ - org/project
diff --git a/tests/fixtures/config/single-tenant/git/layout-footer-message/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-footer-message/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-footer-message/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml
new file mode 100644
index 0000000..0c04070
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml
@@ -0,0 +1,38 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ source:
+ gerrit
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ footer-message: For CI problems and help debugging, contact ci@example.org
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ smtp:
+ to: you@example.com
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ smtp:
+ to: you@example.com
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: project-test1
+# success-url: http://logs.exxxample.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - project-test1
+
diff --git a/tests/fixtures/custom_functions.py b/tests/fixtures/custom_functions.py
deleted file mode 100644
index 4712052..0000000
--- a/tests/fixtures/custom_functions.py
+++ /dev/null
@@ -1,2 +0,0 @@
-def select_debian_node(item, params):
- params['ZUUL_NODE'] = 'debian'
diff --git a/tests/fixtures/custom_functions_live_reconfiguration_functions.py b/tests/fixtures/custom_functions_live_reconfiguration_functions.py
deleted file mode 100644
index d8e06f4..0000000
--- a/tests/fixtures/custom_functions_live_reconfiguration_functions.py
+++ /dev/null
@@ -1,2 +0,0 @@
-def select_debian_node(item, params):
- params['ZUUL_NODE'] = 'wheezy'
diff --git a/tests/fixtures/layout-bad-queue.yaml b/tests/fixtures/layout-bad-queue.yaml
deleted file mode 100644
index 3eb2051..0000000
--- a/tests/fixtures/layout-bad-queue.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
- - name: post
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: ref-updated
- ref: ^(?!refs/).*$
-
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- start:
- gerrit:
- verified: 0
- precedence: high
-
-jobs:
- - name: project1-project2-integration
- queue-name: integration
- - name: project1-test1
- queue-name: not_integration
-
-projects:
- - name: org/project1
- check:
- - project1-merge:
- - project1-test1
- - project1-test2
- - project1-project2-integration
- gate:
- - project1-merge:
- - project1-test1
- - project1-test2
- - project1-project2-integration
- post:
- - project1-post
-
- - name: org/project2
- check:
- - project2-merge:
- - project2-test1
- - project2-test2
- - project1-project2-integration
- gate:
- - project2-merge:
- - project2-test1
- - project2-test2
- - project1-project2-integration
- post:
- - project2-post
diff --git a/tests/fixtures/layout-footer-message.yaml b/tests/fixtures/layout-footer-message.yaml
deleted file mode 100644
index 7977c19..0000000
--- a/tests/fixtures/layout-footer-message.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-includes:
- - python-file: custom_functions.py
-
-pipelines:
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- footer-message: For CI problems and help debugging, contact ci@example.org
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- smtp:
- to: success@example.org
- failure:
- gerrit:
- verified: -2
- smtp:
- to: failure@example.org
- start:
- gerrit:
- verified: 0
- precedence: high
-
-projects:
- - name: org/project
- gate:
- - test1
- - test2
diff --git a/tests/fixtures/layout-live-reconfiguration-functions.yaml b/tests/fixtures/layout-live-reconfiguration-functions.yaml
index e261a88..b22b3ab 100644
--- a/tests/fixtures/layout-live-reconfiguration-functions.yaml
+++ b/tests/fixtures/layout-live-reconfiguration-functions.yaml
@@ -26,12 +26,3 @@
- name: ^.*-merge$
failure-message: Unable to merge change
hold-following-changes: true
- - name: node-project-test1
- parameter-function: select_debian_node
-
-projects:
- - name: org/node-project
- gate:
- - node-project-merge:
- - node-project-test1
- - node-project-test2
diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml
index eb8f17c..6131de0 100644
--- a/tests/fixtures/layout.yaml
+++ b/tests/fixtures/layout.yaml
@@ -129,8 +129,6 @@
- name: project-testfile
files:
- '.*-requires'
- - name: node-project-test1
- parameter-function: select_debian_node
- name: project1-project2-integration
queue-name: integration
- name: mutex-one
@@ -216,12 +214,6 @@
post:
- nonvoting-project-post
- - name: org/node-project
- gate:
- - node-project-merge:
- - node-project-test1
- - node-project-test2
-
- name: org/conflict-project
conflict:
- conflict-project-merge:
diff --git a/tests/fixtures/zuul-connections-multiple-gerrits.conf b/tests/fixtures/zuul-connections-multiple-gerrits.conf
index 89f0aa6..3e6850d 100644
--- a/tests/fixtures/zuul-connections-multiple-gerrits.conf
+++ b/tests/fixtures/zuul-connections-multiple-gerrits.conf
@@ -7,11 +7,14 @@
job_name_in_report=true
[merger]
-git_dir=/tmp/zuul-test/git
+git_dir=/tmp/zuul-test/merger-git
git_user_email=zuul@example.com
git_user_name=zuul
zuul_url=http://zuul.example.com/p
+[launcher]
+git_dir=/tmp/zuul-test/launcher-git
+
[swift]
authurl=https://identity.api.example.org/v2.0/
user=username
diff --git a/tests/fixtures/zuul-connections-same-gerrit.conf b/tests/fixtures/zuul-connections-same-gerrit.conf
index 43109d2..57b5182 100644
--- a/tests/fixtures/zuul-connections-same-gerrit.conf
+++ b/tests/fixtures/zuul-connections-same-gerrit.conf
@@ -7,11 +7,14 @@
job_name_in_report=true
[merger]
-git_dir=/tmp/zuul-test/git
+git_dir=/tmp/zuul-test/merger-git
git_user_email=zuul@example.com
git_user_name=zuul
zuul_url=http://zuul.example.com/p
+[launcher]
+git_dir=/tmp/zuul-test/launcher-git
+
[swift]
authurl=https://identity.api.example.org/v2.0/
user=username
diff --git a/tests/fixtures/zuul-git-driver.conf b/tests/fixtures/zuul-git-driver.conf
new file mode 100644
index 0000000..868e272
--- /dev/null
+++ b/tests/fixtures/zuul-git-driver.conf
@@ -0,0 +1,43 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=config/zuul-connections-same-gerrit/main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[launcher]
+git_dir=/tmp/zuul-test/launcher-git
+
+[swift]
+authurl=https://identity.api.example.org/v2.0/
+user=username
+key=password
+tenant_name=" "
+
+default_container=logs
+region_name=EXP
+logserver_prefix=http://logs.example.org/server.app/
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=none
+
+[connection git]
+driver=git
+baseurl=""
+
+[connection outgoing_smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/fixtures/zuul.conf b/tests/fixtures/zuul.conf
index c08b5ad..48129d8 100644
--- a/tests/fixtures/zuul.conf
+++ b/tests/fixtures/zuul.conf
@@ -7,11 +7,14 @@
job_name_in_report=true
[merger]
-git_dir=/tmp/zuul-test/git
+git_dir=/tmp/zuul-test/merger-git
git_user_email=zuul@example.com
git_user_name=zuul
zuul_url=http://zuul.example.com/p
+[launcher]
+git_dir=/tmp/zuul-test/launcher-git
+
[swift]
authurl=https://identity.api.example.org/v2.0/
user=username
diff --git a/tests/unit/test_git_driver.py b/tests/unit/test_git_driver.py
new file mode 100644
index 0000000..4d75944
--- /dev/null
+++ b/tests/unit/test_git_driver.py
@@ -0,0 +1,42 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tests.base import ZuulTestCase
+
+
+class TestGitDriver(ZuulTestCase):
+ config_file = 'zuul-git-driver.conf'
+ tenant_config_file = 'config/git-driver/main.yaml'
+
+ def setup_config(self):
+ super(TestGitDriver, self).setup_config()
+ self.config.set('connection git', 'baseurl', self.upstream_root)
+
+ def test_git_driver(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ # Check that we have the git source for common-config and the
+ # gerrit source for the project.
+ self.assertEqual('git', tenant.config_repos[0][0].name)
+ self.assertEqual('common-config', tenant.config_repos[0][1].name)
+ self.assertEqual('gerrit', tenant.project_repos[0][0].name)
+ self.assertEqual('org/project', tenant.project_repos[0][1].name)
+
+ # The configuration for this test is accessed via the git
+ # driver (in common-config), rather than the gerrit driver, so
+ # if the job runs, it worked.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual(A.reported, 1)
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index b7dc706..b54eb5f 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -27,6 +27,11 @@
class TestJob(BaseTestCase):
+ def setUp(self):
+ super(TestJob, self).setUp()
+ self.project = model.Project('project', None)
+ self.context = model.SourceContext(self.project, 'master', True)
+
@property
def job(self):
layout = model.Layout()
@@ -54,6 +59,81 @@
self.assertIsNotNone(self.job.voting)
def test_job_inheritance(self):
+ # This is standard job inheritance.
+
+ base_pre = model.PlaybookContext(self.context, 'base-pre')
+ base_run = model.PlaybookContext(self.context, 'base-run')
+ base_post = model.PlaybookContext(self.context, 'base-post')
+
+ base = model.Job('base')
+ base.timeout = 30
+ base.pre_run = [base_pre]
+ base.run = [base_run]
+ base.post_run = [base_post]
+ base.auth = dict(foo='bar', inherit=False)
+
+ py27 = model.Job('py27')
+ self.assertEqual(None, py27.timeout)
+ py27.inheritFrom(base)
+ self.assertEqual(30, py27.timeout)
+ self.assertEqual(['base-pre'],
+ [x.path for x in py27.pre_run])
+ self.assertEqual(['base-run'],
+ [x.path for x in py27.run])
+ self.assertEqual(['base-post'],
+ [x.path for x in py27.post_run])
+ self.assertEqual({}, py27.auth)
+
+ def test_job_variants(self):
+ # This simulates freezing a job.
+
+ py27_pre = model.PlaybookContext(self.context, 'py27-pre')
+ py27_run = model.PlaybookContext(self.context, 'py27-run')
+ py27_post = model.PlaybookContext(self.context, 'py27-post')
+
+ py27 = model.Job('py27')
+ py27.timeout = 30
+ py27.pre_run = [py27_pre]
+ py27.run = [py27_run]
+ py27.post_run = [py27_post]
+ auth = dict(foo='bar', inherit=False)
+ py27.auth = auth
+
+ job = py27.copy()
+ self.assertEqual(30, job.timeout)
+
+ # Apply the diablo variant
+ diablo = model.Job('py27')
+ diablo.timeout = 40
+ job.applyVariant(diablo)
+
+ self.assertEqual(40, job.timeout)
+ self.assertEqual(['py27-pre'],
+ [x.path for x in job.pre_run])
+ self.assertEqual(['py27-run'],
+ [x.path for x in job.run])
+ self.assertEqual(['py27-post'],
+ [x.path for x in job.post_run])
+ self.assertEqual(auth, job.auth)
+
+ # Set the job to final for the following checks
+ job.final = True
+ self.assertTrue(job.voting)
+
+ good_final = model.Job('py27')
+ good_final.voting = False
+ job.applyVariant(good_final)
+ self.assertFalse(job.voting)
+
+ bad_final = model.Job('py27')
+ bad_final.timeout = 600
+ with testtools.ExpectedException(
+ Exception,
+ "Unable to modify final job"):
+ job.applyVariant(bad_final)
+
+ def test_job_inheritance_configloader(self):
+ # TODO(jeblair): move this to a configloader test
layout = model.Layout()
pipeline = model.Pipeline('gate', layout)
@@ -66,6 +146,8 @@
'_source_context': context,
'name': 'base',
'timeout': 30,
+ 'pre-run': 'base-pre',
+ 'post-run': 'base-post',
'nodes': [{
'name': 'controller',
'image': 'base',
@@ -76,6 +158,8 @@
'_source_context': context,
'name': 'python27',
'parent': 'base',
+ 'pre-run': 'py27-pre',
+ 'post-run': 'py27-post',
'nodes': [{
'name': 'controller',
'image': 'new',
@@ -89,6 +173,9 @@
'branches': [
'stable/diablo'
],
+ 'pre-run': 'py27-diablo-pre',
+ 'run': 'py27-diablo',
+ 'post-run': 'py27-diablo-post',
'nodes': [{
'name': 'controller',
'image': 'old',
@@ -97,6 +184,17 @@
})
layout.addJob(python27diablo)
+ python27essex = configloader.JobParser.fromYaml(layout, {
+ '_source_context': context,
+ 'name': 'python27',
+ 'branches': [
+ 'stable/essex'
+ ],
+ 'pre-run': 'py27-essex-pre',
+ 'post-run': 'py27-essex-post',
+ })
+ layout.addJob(python27essex)
+
project_config = configloader.ProjectParser.fromYaml(layout, {
'_source_context': context,
'name': 'project',
@@ -117,6 +215,7 @@
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change))
+ self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
@@ -126,6 +225,15 @@
nodes = job.nodeset.getNodes()
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].image, 'new')
+ self.assertEqual([x.path for x in job.pre_run],
+ ['playbooks/base-pre',
+ 'playbooks/py27-pre'])
+ self.assertEqual([x.path for x in job.post_run],
+ ['playbooks/py27-post',
+ 'playbooks/base-post'])
+ self.assertEqual([x.path for x in job.run],
+ ['playbooks/python27',
+ 'playbooks/base'])
# Test diablo
change.branch = 'stable/diablo'
@@ -135,6 +243,7 @@
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertTrue(python27diablo.changeMatches(change))
+ self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
@@ -144,6 +253,42 @@
nodes = job.nodeset.getNodes()
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].image, 'old')
+ self.assertEqual([x.path for x in job.pre_run],
+ ['playbooks/base-pre',
+ 'playbooks/py27-pre',
+ 'playbooks/py27-diablo-pre'])
+ self.assertEqual([x.path for x in job.post_run],
+ ['playbooks/py27-diablo-post',
+ 'playbooks/py27-post',
+ 'playbooks/base-post'])
+ self.assertEqual([x.path for x in job.run],
+ ['playbooks/py27-diablo']),
+
+ # Test essex
+ change.branch = 'stable/essex'
+ item = queue.enqueueChange(change)
+ item.current_build_set.layout = layout
+
+ self.assertTrue(base.changeMatches(change))
+ self.assertTrue(python27.changeMatches(change))
+ self.assertFalse(python27diablo.changeMatches(change))
+ self.assertTrue(python27essex.changeMatches(change))
+
+ item.freezeJobTree()
+ self.assertEqual(len(item.getJobs()), 1)
+ job = item.getJobs()[0]
+ self.assertEqual(job.name, 'python27')
+ self.assertEqual([x.path for x in job.pre_run],
+ ['playbooks/base-pre',
+ 'playbooks/py27-pre',
+ 'playbooks/py27-essex-pre'])
+ self.assertEqual([x.path for x in job.post_run],
+ ['playbooks/py27-essex-post',
+ 'playbooks/py27-post',
+ 'playbooks/base-post'])
+ self.assertEqual([x.path for x in job.run],
+ ['playbooks/python27',
+ 'playbooks/base'])
def test_job_auth_inheritance(self):
layout = model.Layout()
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index db458d4..03aff00 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -56,6 +56,7 @@
self.assertEqual(A.reported, 2)
self.assertEqual(self.getJobFromHistory('project-test1').node,
'image1')
+ self.assertIsNone(self.getJobFromHistory('project-test2').node)
# TODOv3(jeblair): we may want to report stats by tenant (also?).
self.assertReportedStat('gerrit.event.comment-added', value='1|c')
@@ -1272,6 +1273,7 @@
self.assertEqual(self.getJobFromHistory('project-test2').result,
'FAILURE')
+ @skip("This test generally works but times out frequently")
def test_dependent_behind_dequeue(self):
"test that dependent changes behind dequeued changes work"
# This complicated test is a reproduction of a real life bug
@@ -1376,8 +1378,12 @@
self.assertEmptyQueues()
self.build_history = []
- path = os.path.join(self.git_root, "org/project")
- print(repack_repo(path))
+ path = os.path.join(self.merger_git_root, "org/project")
+ if os.path.exists(path):
+ repack_repo(path)
+ path = os.path.join(self.launcher_git_root, "org/project")
+ if os.path.exists(path):
+ repack_repo(path)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
@@ -1403,9 +1409,13 @@
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
A.addPatchset(large=True)
path = os.path.join(self.upstream_root, "org/project1")
- print(repack_repo(path))
- path = os.path.join(self.git_root, "org/project1")
- print(repack_repo(path))
+ repack_repo(path)
+ path = os.path.join(self.merger_git_root, "org/project1")
+ if os.path.exists(path):
+ repack_repo(path)
+ path = os.path.join(self.launcher_git_root, "org/project1")
+ if os.path.exists(path):
+ repack_repo(path)
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
@@ -2068,21 +2078,16 @@
self.sched.testConfig(self.config.get('zuul', 'tenant_config'),
self.connections)
- @skip("Disabled for early v3 development")
def test_queue_names(self):
"Test shared change queue names"
- project1 = self.sched.layout.projects['org/project1']
- project2 = self.sched.layout.projects['org/project2']
- q1 = self.sched.layout.pipelines['gate'].getQueue(project1)
- q2 = self.sched.layout.pipelines['gate'].getQueue(project2)
- self.assertEqual(q1.name, 'integration')
- self.assertEqual(q2.name, 'integration')
-
- self.updateConfigLayout(
- 'tests/fixtures/layout-bad-queue.yaml')
- with testtools.ExpectedException(
- Exception, "More than one name assigned to change queue"):
- self.sched.reconfigure(self.config)
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ source = tenant.layout.pipelines['gate'].source
+ project1 = source.getProject('org/project1')
+ project2 = source.getProject('org/project2')
+ q1 = tenant.layout.pipelines['gate'].getQueue(project1)
+ q2 = tenant.layout.pipelines['gate'].getQueue(project2)
+ self.assertEqual(q1.name, 'integrated')
+ self.assertEqual(q2.name, 'integrated')
def test_queue_precedence(self):
"Test that queue precedence works"
@@ -2245,21 +2250,6 @@
self.assertEqual(B.reported, 1)
self.assertFalse('test-mutex' in self.sched.mutex.mutexes)
- @skip("Disabled for early v3 development")
- def test_node_label(self):
- "Test that a job runs on a specific node label"
- self.worker.registerFunction('build:node-project-test1:debian')
-
- A = self.fake_gerrit.addFakeChange('org/node-project', 'master', 'A')
- A.addApproval('code-review', 2)
- self.fake_gerrit.addEvent(A.addApproval('approved', 1))
- self.waitUntilSettled()
-
- self.assertIsNone(self.getJobFromHistory('node-project-merge').node)
- self.assertEqual(self.getJobFromHistory('node-project-test1').node,
- 'debian')
- self.assertIsNone(self.getJobFromHistory('node-project-test2').node)
-
def test_live_reconfiguration(self):
"Test that live reconfiguration works"
self.launch_server.hold_jobs_in_build = True
@@ -2682,7 +2672,11 @@
self.assertEqual(A.reported, 2)
# Delete org/new-project zuul repo. Should be recloned.
- shutil.rmtree(os.path.join(self.git_root, "org/delete-project"))
+ p = 'org/delete-project'
+ if os.path.exists(os.path.join(self.merger_git_root, p)):
+ shutil.rmtree(os.path.join(self.merger_git_root, p))
+ if os.path.exists(os.path.join(self.launcher_git_root, p)):
+ shutil.rmtree(os.path.join(self.launcher_git_root, p))
B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')
@@ -2901,7 +2895,6 @@
self.assertEqual(A.reported, 2)
self.assertEqual(r, True)
- @skip("Disabled for early v3 development")
def test_client_enqueue_ref(self):
"Test that the RPC client can enqueue a ref"
@@ -3356,17 +3349,14 @@
self.launch_server.release()
self.waitUntilSettled()
- @skip("Disabled for early v3 development")
def test_footer_message(self):
"Test a pipeline's footer message is correctly added to the report."
- self.updateConfigLayout(
- 'tests/fixtures/layout-footer-message.yaml')
+ self.updateConfigLayout('layout-footer-message')
self.sched.reconfigure(self.config)
- self.registerJobs()
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
- self.launch_server.failJob('test1', A)
+ self.launch_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
@@ -3377,25 +3367,17 @@
self.assertEqual(2, len(self.smtp_messages))
- failure_body = """\
+ failure_msg = """\
Build failed. For information on how to proceed, see \
-http://wiki.example.org/Test_Failures
+http://wiki.example.org/Test_Failures"""
-- test1 http://logs.example.com/1/1/gate/test1/0 : FAILURE in 0s
-- test2 http://logs.example.com/1/1/gate/test2/1 : SUCCESS in 0s
-
+ footer_msg = """\
For CI problems and help debugging, contact ci@example.org"""
- success_body = """\
-Build succeeded.
-
-- test1 http://logs.example.com/2/1/gate/test1/2 : SUCCESS in 0s
-- test2 http://logs.example.com/2/1/gate/test2/3 : SUCCESS in 0s
-
-For CI problems and help debugging, contact ci@example.org"""
-
- self.assertEqual(failure_body, self.smtp_messages[0]['body'])
- self.assertEqual(success_body, self.smtp_messages[1]['body'])
+ self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))
+ self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))
+ self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))
+ self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))
@skip("Disabled for early v3 development")
def test_merge_failure_reporters(self):
diff --git a/zuul/change_matcher.py b/zuul/change_matcher.py
index ca2d93f..845ba1c 100644
--- a/zuul/change_matcher.py
+++ b/zuul/change_matcher.py
@@ -35,9 +35,15 @@
def copy(self):
return self.__class__(self._regex)
+ def __deepcopy__(self, memo):
+ return self.copy()
+
def __eq__(self, other):
return str(self) == str(other)
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
def __str__(self):
return '{%s:%s}' % (self.__class__.__name__, self._regex)
diff --git a/zuul/cmd/launcher.py b/zuul/cmd/launcher.py
index 49643ae..596fd1a 100644
--- a/zuul/cmd/launcher.py
+++ b/zuul/cmd/launcher.py
@@ -29,7 +29,7 @@
import signal
import zuul.cmd
-import zuul.launcher.ansiblelaunchserver
+import zuul.launcher.server
# No zuul imports that pull in paramiko here; it must not be
# imported until after the daemonization.
@@ -52,7 +52,7 @@
action='store_true',
help='keep local jobdirs after run completes')
parser.add_argument('command',
- choices=zuul.launcher.ansiblelaunchserver.COMMANDS,
+ choices=zuul.launcher.server.COMMANDS,
nargs='?')
self.args = parser.parse_args()
@@ -79,8 +79,8 @@
self.log = logging.getLogger("zuul.Launcher")
- LaunchServer = zuul.launcher.ansiblelaunchserver.LaunchServer
- self.launcher = LaunchServer(self.config,
+ LaunchServer = zuul.launcher.server.LaunchServer
+ self.launcher = LaunchServer(self.config, self.connections,
keep_jobdir=self.args.keep_jobdir)
self.launcher.start()
@@ -102,7 +102,7 @@
server.parse_arguments()
server.read_config()
- if server.args.command in zuul.launcher.ansiblelaunchserver.COMMANDS:
+ if server.args.command in zuul.launcher.server.COMMANDS:
server.send_command(server.args.command)
sys.exit(0)
diff --git a/zuul/cmd/server.py b/zuul/cmd/scheduler.py
similarity index 92%
rename from zuul/cmd/server.py
rename to zuul/cmd/scheduler.py
index ff9f2d9..e5497dc 100755
--- a/zuul/cmd/server.py
+++ b/zuul/cmd/scheduler.py
@@ -35,9 +35,9 @@
# Similar situation with gear and statsd.
-class Server(zuul.cmd.ZuulApp):
+class Scheduler(zuul.cmd.ZuulApp):
def __init__(self):
- super(Server, self).__init__()
+ super(Scheduler, self).__init__()
self.gear_server_pid = None
def parse_arguments(self):
@@ -160,7 +160,7 @@
self.start_gear_server()
self.setup_logging('zuul', 'log_config')
- self.log = logging.getLogger("zuul.Server")
+ self.log = logging.getLogger("zuul.Scheduler")
self.sched = zuul.scheduler.Scheduler(self.config)
# TODO(jhesketh): Move swift into a connection?
@@ -218,31 +218,31 @@
def main():
- server = Server()
- server.parse_arguments()
+ scheduler = Scheduler()
+ scheduler.parse_arguments()
- server.read_config()
+ scheduler.read_config()
- if server.args.layout:
- server.config.set('zuul', 'layout_config', server.args.layout)
+ if scheduler.args.layout:
+ scheduler.config.set('zuul', 'layout_config', scheduler.args.layout)
- if server.args.validate:
- path = server.args.validate
+ if scheduler.args.validate:
+ path = scheduler.args.validate
if path is True:
path = None
- sys.exit(server.test_config(path))
+ sys.exit(scheduler.test_config(path))
- if server.config.has_option('zuul', 'pidfile'):
- pid_fn = os.path.expanduser(server.config.get('zuul', 'pidfile'))
+ if scheduler.config.has_option('zuul', 'pidfile'):
+ pid_fn = os.path.expanduser(scheduler.config.get('zuul', 'pidfile'))
else:
pid_fn = '/var/run/zuul/zuul.pid'
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
- if server.args.nodaemon:
- server.main()
+ if scheduler.args.nodaemon:
+ scheduler.main()
else:
with daemon.DaemonContext(pidfile=pid):
- server.main()
+ scheduler.main()
if __name__ == "__main__":
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 885e6b3..7a07956 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -103,27 +103,66 @@
'attempts': int,
'pre-run': to_list(str),
'post-run': to_list(str),
+ 'run': str,
'_source_context': model.SourceContext,
}
return vs.Schema(job)
+ simple_attributes = [
+ 'timeout',
+ 'workspace',
+ 'voting',
+ 'hold-following-changes',
+ 'mutex',
+ 'attempts',
+ 'failure-message',
+ 'success-message',
+ 'failure-url',
+ 'success-url',
+ ]
+
@staticmethod
def fromYaml(layout, conf):
JobParser.getSchema()(conf)
+ # NB: The default detection system in the Job class requires
+ # that we always assign values directly rather than modifying
+ # them (e.g., "job.run = ..." rather than
+ # "job.run.append(...)").
+
job = model.Job(conf['name'])
+ job.source_context = conf.get('_source_context')
if 'auth' in conf:
job.auth = conf.get('auth')
+
if 'parent' in conf:
parent = layout.getJob(conf['parent'])
- job.inheritFrom(parent, 'parent while parsing')
- job.timeout = conf.get('timeout', job.timeout)
- job.workspace = conf.get('workspace', job.workspace)
- job.voting = conf.get('voting', True)
- job.hold_following_changes = conf.get('hold-following-changes', False)
- job.mutex = conf.get('mutex', None)
- job.attempts = conf.get('attempts', 3)
+ job.inheritFrom(parent)
+
+ for pre_run_name in as_list(conf.get('pre-run')):
+ full_pre_run_name = os.path.join('playbooks', pre_run_name)
+ pre_run = model.PlaybookContext(job.source_context,
+ full_pre_run_name)
+ job.pre_run = job.pre_run + (pre_run,)
+ for post_run_name in as_list(conf.get('post-run')):
+ full_post_run_name = os.path.join('playbooks', post_run_name)
+ post_run = model.PlaybookContext(job.source_context,
+ full_post_run_name)
+ job.post_run = (post_run,) + job.post_run
+ if 'run' in conf:
+ run_name = os.path.join('playbooks', conf['run'])
+ run = model.PlaybookContext(job.source_context, run_name)
+ job.run = (run,)
+ else:
+ run_name = os.path.join('playbooks', job.name)
+ run = model.PlaybookContext(job.source_context, run_name)
+ job.implied_run = (run,) + job.implied_run
+
+ for k in JobParser.simple_attributes:
+ a = k.replace('-', '_')
+ if k in conf:
+ setattr(job, a, conf[k])
if 'nodes' in conf:
conf_nodes = conf['nodes']
if isinstance(conf_nodes, six.string_types):
@@ -140,37 +179,8 @@
if tags:
# Tags are merged via a union rather than a
# destructive copy because they are intended to
- # accumulate onto any previously applied tags from
- # metajobs.
+ # accumulate onto any previously applied tags.
job.tags = job.tags.union(set(tags))
- # The source attribute and playbook info may not be
- # overridden -- they are always supplied by the config loader.
- # They correspond to the Project instance of the repo where it
- # originated, and the branch name.
- job.source_context = conf.get('_source_context')
- pre_run_name = conf.get('pre-run')
- # Append the pre-run command
- if pre_run_name:
- pre_run_name = os.path.join('playbooks', pre_run_name)
- pre_run = model.PlaybookContext(job.source_context,
- pre_run_name)
- job.pre_run.append(pre_run)
- # Prepend the post-run command
- post_run_name = conf.get('post-run')
- if post_run_name:
- post_run_name = os.path.join('playbooks', post_run_name)
- post_run = model.PlaybookContext(job.source_context,
- post_run_name)
- job.post_run.insert(0, post_run)
- # Set the run command
- run_name = job.name
- run_name = os.path.join('playbooks', run_name)
- run = model.PlaybookContext(job.source_context, run_name)
- job.run = run
- job.failure_message = conf.get('failure-message', job.failure_message)
- job.success_message = conf.get('success-message', job.success_message)
- job.failure_url = conf.get('failure-url', job.failure_url)
- job.success_url = conf.get('success-url', job.success_url)
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
@@ -240,7 +250,8 @@
tree = model.JobTree(None)
for conf_job in conf:
if isinstance(conf_job, six.string_types):
- tree.addJob(model.Job(conf_job))
+ job = model.Job(conf_job)
+ tree.addJob(job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params, or
# be the root of a sub job tree, or both.
@@ -252,8 +263,9 @@
attrs['_source_context'] = source_context
subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
else:
- # Not overriding, so get existing job
- subtree = tree.addJob(layout.getJob(jobname))
+ # Not overriding, so add a blank job
+ job = model.Job(jobname)
+ subtree = tree.addJob(job)
if jobs:
# This is the root of a sub tree
@@ -313,8 +325,7 @@
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_tree.inheritFrom(
- template_pipeline.job_tree,
- 'job tree while parsing')
+ template_pipeline.job_tree)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if queue_name:
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 683c8ff..9c54b4c 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -106,13 +106,19 @@
'comment-added': 'author',
'ref-updated': 'submitter',
'reviewer-added': 'reviewer', # Gerrit 2.5/2.6
+ 'ref-replicated': None,
+ 'ref-replication-done': None,
+ 'topic-changed': 'changer',
}
- try:
- event.account = data.get(accountfield_from_type[event.type])
- except KeyError:
- self.log.warning("Received unrecognized event type '%s' from Gerrit.\
- Can not get account information." % event.type)
- event.account = None
+ event.account = None
+ if event.type in accountfield_from_type:
+ field = accountfield_from_type[event.type]
+ if field:
+ event.account = data.get(accountfield_from_type[event.type])
+ else:
+ self.log.warning("Received unrecognized event type '%s' "
+ "from Gerrit. Can not get account information." %
+ (event.type,))
if event.change_number:
# TODO(jhesketh): Check if the project exists?
@@ -281,7 +287,6 @@
change.newrev = event.newrev
change.url = self._getGitwebUrl(project, sha=event.newrev)
else:
- # TODOv3(jeblair): we need to get the project from the event
project = self.getProject(event.project_name)
change = NullChange(project)
return change
@@ -754,7 +759,7 @@
project.name)
return url
- def getGitwebUrl(self, project, sha=None):
+ def _getGitwebUrl(self, project, sha=None):
url = '%s/gitweb?p=%s.git' % (self.baseurl, project)
if sha:
url += ';a=commitdiff;h=' + sha
diff --git a/zuul/driver/gerrit/gerritsource.py b/zuul/driver/gerrit/gerritsource.py
index 8b03135..c5e46b1 100644
--- a/zuul/driver/gerrit/gerritsource.py
+++ b/zuul/driver/gerrit/gerritsource.py
@@ -48,4 +48,4 @@
return self.connection.getGitUrl(project)
def _getGitwebUrl(self, project, sha=None):
- return self.connection.getGitwebUrl(project, sha)
+ return self.connection._getGitwebUrl(project, sha)
diff --git a/zuul/driver/git/__init__.py b/zuul/driver/git/__init__.py
new file mode 100644
index 0000000..abedf6a
--- /dev/null
+++ b/zuul/driver/git/__init__.py
@@ -0,0 +1,27 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from zuul.driver import Driver, ConnectionInterface, SourceInterface
+import gitconnection
+import gitsource
+
+
+class GitDriver(Driver, ConnectionInterface, SourceInterface):
+ name = 'git'
+
+ def getConnection(self, name, config):
+ return gitconnection.GitConnection(self, name, config)
+
+ def getSource(self, connection):
+ return gitsource.GitSource(self, connection)
diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py
new file mode 100644
index 0000000..e72cc77
--- /dev/null
+++ b/zuul/driver/git/gitconnection.py
@@ -0,0 +1,54 @@
+# Copyright 2011 OpenStack, LLC.
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+
+from zuul.connection import BaseConnection
+from zuul.model import Project
+
+
+class GitConnection(BaseConnection):
+ driver_name = 'git'
+ log = logging.getLogger("connection.git")
+
+ def __init__(self, driver, connection_name, connection_config):
+ super(GitConnection, self).__init__(driver, connection_name,
+ connection_config)
+ if 'baseurl' not in self.connection_config:
+ raise Exception('baseurl is required for git connections in '
+ '%s' % self.connection_name)
+
+ self.baseurl = self.connection_config.get('baseurl')
+ self.projects = {}
+
+ def getProject(self, name):
+ if name not in self.projects:
+ self.projects[name] = Project(name, self.connection_name)
+ return self.projects[name]
+
+ def getProjectBranches(self, project):
+ # TODO(jeblair): implement; this will need to handle local or
+ # remote git urls.
+ raise NotImplemented()
+
+ def getGitUrl(self, project):
+ url = '%s/%s' % (self.baseurl, project.name)
+ return url
+
+
+def getSchema():
+ git_connection = v.Any(str, v.Schema({}, extra=True))
+ return git_connection
diff --git a/zuul/driver/git/gitsource.py b/zuul/driver/git/gitsource.py
new file mode 100644
index 0000000..bbe799a
--- /dev/null
+++ b/zuul/driver/git/gitsource.py
@@ -0,0 +1,45 @@
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+from zuul.source import BaseSource
+
+
+class GitSource(BaseSource):
+ name = 'git'
+ log = logging.getLogger("zuul.source.Git")
+
+ def getRefSha(self, project, ref):
+ raise NotImplemented()
+
+ def isMerged(self, change, head=None):
+ raise NotImplemented()
+
+ def canMerge(self, change, allow_needs):
+ raise NotImplemented()
+
+ def getChange(self, event, refresh=False):
+ raise NotImplemented()
+
+ def getProject(self, name):
+ return self.connection.getProject(name)
+
+ def getProjectBranches(self, project):
+ return self.connection.getProjectBranches(project)
+
+ def getGitUrl(self, project):
+ return self.connection.getGitUrl(project)
+
+ def getProjectOpenChanges(self, project):
+ raise NotImplemented()
diff --git a/zuul/launcher/client.py b/zuul/launcher/client.py
index 5b60506..ab089b7 100644
--- a/zuul/launcher/client.py
+++ b/zuul/launcher/client.py
@@ -143,49 +143,6 @@
if build.__gearman_job.handle == handle:
self.__zuul_gearman.onUnknownJob(job)
- def waitForGearmanToSettle(self):
- # If we're running the internal gearman server, it's possible
- # that after a restart or reload, we may be immediately ready
- # to run jobs but all the gearman workers may not have
- # registered yet. Give them a sporting chance to show up
- # before we start declaring jobs lost because we don't have
- # gearman functions registered for them.
-
- # Spend up to 30 seconds after we connect to the gearman
- # server waiting for the set of defined jobs to become
- # consistent over a sliding 5 second window.
-
- self.log.info("Waiting for connection to internal Gearman server")
- self.waitForServer()
- self.log.info("Waiting for gearman function set to settle")
- start = time.time()
- last_change = start
- all_functions = set()
- while time.time() - start < 30:
- now = time.time()
- last_functions = set()
- for connection in self.active_connections:
- try:
- req = gear.StatusAdminRequest()
- connection.sendAdminRequest(req, timeout=300)
- except Exception:
- self.log.exception("Exception while checking functions")
- continue
- for line in req.response.split('\n'):
- parts = [x.strip() for x in line.split()]
- if not parts or parts[0] == '.':
- continue
- last_functions.add(parts[0])
- if last_functions != all_functions:
- last_change = now
- all_functions.update(last_functions)
- else:
- if now - last_change > 5:
- self.log.info("Gearman function set has settled")
- break
- time.sleep(1)
- self.log.info("Done waiting for Gearman server")
-
class LaunchClient(object):
log = logging.getLogger("zuul.LaunchClient")
@@ -207,10 +164,6 @@
self.gearman = ZuulGearmanClient(self)
self.gearman.addServer(server, port)
- if (config.has_option('gearman_server', 'start') and
- config.getboolean('gearman_server', 'start')):
- self.gearman.waitForGearmanToSettle()
-
self.cleanup_thread = GearmanCleanup(self)
self.cleanup_thread.start()
self.function_cache = set()
@@ -378,7 +331,7 @@
params['projects'] = []
if job.name != 'noop':
- params['playbook'] = job.run.toDict()
+ params['playbooks'] = [x.toDict() for x in job.run]
params['pre_playbooks'] = [x.toDict() for x in job.pre_run]
params['post_playbooks'] = [x.toDict() for x in job.post_run]
diff --git a/zuul/launcher/server.py b/zuul/launcher/server.py
index 4e0fdd2..df71cc9 100644
--- a/zuul/launcher/server.py
+++ b/zuul/launcher/server.py
@@ -28,13 +28,17 @@
import gear
-import zuul.merger
+import zuul.merger.merger
import zuul.ansible.library
from zuul.lib import commandsocket
ANSIBLE_WATCHDOG_GRACE = 5 * 60
+COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
+ 'unverbose']
+
+
class Watchdog(object):
def __init__(self, timeout, function, args):
self.timeout = timeout
@@ -84,9 +88,8 @@
self.known_hosts = os.path.join(self.ansible_root, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory')
self.vars = os.path.join(self.ansible_root, 'vars.yaml')
- self.playbook_root = os.path.join(self.ansible_root, 'playbook')
- os.makedirs(self.playbook_root)
- self.playbook = JobDirPlaybook(self.playbook_root)
+ self.playbooks = [] # The list of candidate playbooks
+ self.playbook = None # A pointer to the candidate we have chosen
self.pre_playbooks = []
self.post_playbooks = []
self.config = os.path.join(self.ansible_root, 'ansible.cfg')
@@ -108,6 +111,14 @@
self.post_playbooks.append(playbook)
return playbook
+ def addPlaybook(self):
+ count = len(self.playbooks)
+ root = os.path.join(self.ansible_root, 'playbook_%i' % (count,))
+ os.makedirs(root)
+ playbook = JobDirPlaybook(root)
+ self.playbooks.append(playbook)
+ return playbook
+
def cleanup(self):
if not self.keep:
shutil.rmtree(self.root)
@@ -197,10 +208,10 @@
unverbose=self.verboseOff,
)
- if self.config.has_option('merger', 'git_dir'):
- self.merge_root = self.config.get('merger', 'git_dir')
+ if self.config.has_option('launcher', 'git_dir'):
+ self.merge_root = self.config.get('launcher', 'git_dir')
else:
- self.merge_root = '/var/lib/zuul/git'
+ self.merge_root = '/var/lib/zuul/launcher-git'
if self.config.has_option('merger', 'git_user_email'):
self.merge_email = self.config.get('merger', 'git_user_email')
@@ -280,6 +291,7 @@
self.worker.shutdown()
self._command_running = False
self.command_socket.stop()
+ self.update_queue.put(None)
self.log.debug("Stopped")
def pause(self):
@@ -325,6 +337,9 @@
def _innerUpdateLoop(self):
# Inside of a loop that keeps the main repository up to date
task = self.update_queue.get()
+ if task is None:
+ # We are asked to stop
+ return
self.log.info("Updating repo %s from %s" % (task.project, task.url))
self.merger.updateRepo(task.project, task.url)
self.log.debug("Finished updating repo %s from %s" %
@@ -360,6 +375,8 @@
except Exception:
self.log.exception("Exception while running job")
job.sendWorkException(traceback.format_exc())
+ except gear.InterruptedError:
+ pass
except Exception:
self.log.exception("Exception while getting job")
@@ -563,26 +580,38 @@
hosts.append((node['name'], dict(ansible_connection='local')))
return hosts
- def findPlaybook(self, path):
+ def findPlaybook(self, path, required=False):
for ext in ['.yaml', '.yml']:
fn = path + ext
if os.path.exists(fn):
return fn
- raise Exception("Unable to find playbook %s" % path)
+ if required:
+ raise Exception("Unable to find playbook %s" % path)
+ return None
def preparePlaybookRepos(self, args):
for playbook in args['pre_playbooks']:
jobdir_playbook = self.jobdir.addPrePlaybook()
- self.preparePlaybookRepo(jobdir_playbook, playbook, args)
+ self.preparePlaybookRepo(jobdir_playbook, playbook,
+ args, main=False)
- jobdir_playbook = self.jobdir.playbook
- self.preparePlaybookRepo(jobdir_playbook, args['playbook'], args)
+ for playbook in args['playbooks']:
+ jobdir_playbook = self.jobdir.addPlaybook()
+ self.preparePlaybookRepo(jobdir_playbook, playbook,
+ args, main=True)
+ if jobdir_playbook.path is not None:
+ self.jobdir.playbook = jobdir_playbook
+ break
+ if self.jobdir.playbook is None:
+ raise Exception("No valid playbook found")
for playbook in args['post_playbooks']:
jobdir_playbook = self.jobdir.addPostPlaybook()
- self.preparePlaybookRepo(jobdir_playbook, playbook, args)
+ self.preparePlaybookRepo(jobdir_playbook, playbook,
+ args, main=False)
- def preparePlaybookRepo(self, jobdir_playbook, playbook, args):
+ def preparePlaybookRepo(self, jobdir_playbook, playbook, args, main):
+ self.log.debug("Prepare playbook repo for %s" % (playbook,))
# Check out the playbook repo if needed and set the path to
# the playbook that should be run.
jobdir_playbook.secure = playbook['secure']
@@ -602,7 +631,7 @@
path = os.path.join(self.jobdir.git_root,
project.name,
playbook['path'])
- jobdir_playbook.path = self.findPlaybook(path)
+ jobdir_playbook.path = self.findPlaybook(path, main)
return
# The playbook repo is either a config repo, or it isn't in
# the stack of changes we are testing, so check out the branch
@@ -614,7 +643,7 @@
path = os.path.join(jobdir_playbook.root,
project.name,
playbook['path'])
- jobdir_playbook.path = self.findPlaybook(path)
+ jobdir_playbook.path = self.findPlaybook(path, main)
def prepareAnsibleFiles(self, args):
with open(self.jobdir.inventory, 'w') as inventory:
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 0a8be03..c8b61a9 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -16,6 +16,7 @@
import zuul.driver.zuul
import zuul.driver.gerrit
+import zuul.driver.git
import zuul.driver.smtp
import zuul.driver.timer
from zuul.connection import BaseConnection
@@ -34,6 +35,7 @@
self.registerDriver(zuul.driver.zuul.ZuulDriver())
self.registerDriver(zuul.driver.gerrit.GerritDriver())
+ self.registerDriver(zuul.driver.git.GitDriver())
self.registerDriver(zuul.driver.smtp.SMTPDriver())
self.registerDriver(zuul.driver.timer.TimerDriver())
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 95028e5..6e5f567 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -318,8 +318,8 @@
"ignoring" % change)
return True
- self.log.debug("Adding change %s to queue %s" %
- (change, change_queue))
+ self.log.info("Adding change %s to queue %s in %s" %
+ (change, change_queue, self.pipeline))
item = change_queue.enqueueChange(change)
if enqueue_time:
item.enqueue_time = enqueue_time
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 750d560..ecce2cf 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -32,7 +32,7 @@
if self.config.has_option('merger', 'git_dir'):
merge_root = self.config.get('merger', 'git_dir')
else:
- merge_root = '/var/lib/zuul/git'
+ merge_root = '/var/lib/zuul/merger-git'
if self.config.has_option('merger', 'git_user_email'):
merge_email = self.config.get('merger', 'git_user_email')
diff --git a/zuul/model.py b/zuul/model.py
index ae8ec17..5a9e367 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -541,6 +541,12 @@
self.branch,
self.secure)
+ def __deepcopy__(self, memo):
+ return self.copy()
+
+ def copy(self):
+ return self.__class__(self.project, self.branch, self.secure)
+
def __ne__(self, other):
return not self.__eq__(other)
@@ -589,20 +595,20 @@
class Job(object):
- """A Job represents the defintion of actions to perform."""
+ """A Job represents the defintion of actions to perform.
+
+ NB: Do not modify attributes of this class, set them directly
+ (e.g., "job.run = ..." rather than "job.run.append(...)").
+ """
def __init__(self, name):
- self.attributes = dict(
- timeout=None,
- # variables={},
- nodeset=NodeSet(),
- auth={},
- workspace=None,
- pre_run=[],
- post_run=[],
- run=None,
- voting=None,
- hold_following_changes=None,
+ # These attributes may override even the final form of a job
+ # in the context of a project-pipeline. They can not affect
+ # the execution of the job, but only whether the job is run
+ # and how it is reported.
+ self.context_attributes = dict(
+ voting=True,
+ hold_following_changes=False,
failure_message=None,
success_message=None,
failure_url=None,
@@ -612,16 +618,44 @@
branch_matcher=None,
file_matcher=None,
irrelevant_file_matcher=None, # skip-if
- tags=set(),
- mutex=None,
- attempts=3,
- source_context=None,
- inheritance_path=[],
+ tags=frozenset(),
)
+ # These attributes affect how the job is actually run and more
+ # care must be taken when overriding them. If a job is
+ # declared "final", these may not be overriden in a
+ # project-pipeline.
+ self.execution_attributes = dict(
+ timeout=None,
+ # variables={},
+ nodeset=NodeSet(),
+ auth={},
+ workspace=None,
+ pre_run=(),
+ post_run=(),
+ run=(),
+ implied_run=(),
+ mutex=None,
+ attempts=3,
+ final=False,
+ )
+
+ # These are generally internal attributes which are not
+ # accessible via configuration.
+ self.other_attributes = dict(
+ name=None,
+ source_context=None,
+ inheritance_path=(),
+ )
+
+ self.inheritable_attributes = {}
+ self.inheritable_attributes.update(self.context_attributes)
+ self.inheritable_attributes.update(self.execution_attributes)
+ self.attributes = {}
+ self.attributes.update(self.inheritable_attributes)
+ self.attributes.update(self.other_attributes)
+
self.name = name
- for k, v in self.attributes.items():
- setattr(self, k, v)
def __ne__(self, other):
return not self.__eq__(other)
@@ -647,24 +681,82 @@
self.branch_matcher,
self.source_context)
- def inheritFrom(self, other, comment='unknown'):
+ def __getattr__(self, name):
+ v = self.__dict__.get(name)
+ if v is None:
+ return copy.deepcopy(self.attributes[name])
+ return v
+
+ def _get(self, name):
+ return self.__dict__.get(name)
+
+ def setRun(self):
+ if not self.run:
+ self.run = self.implied_run
+
+ def inheritFrom(self, other):
"""Copy the inheritable attributes which have been set on the other
job to this job."""
+ if not isinstance(other, Job):
+ raise Exception("Job unable to inherit from %s" % (other,))
+
+ do_not_inherit = set()
+ if other.auth and not other.auth.get('inherit'):
+ do_not_inherit.add('auth')
+
+ # copy all attributes
+ for k in self.inheritable_attributes:
+ if (other._get(k) is not None and k not in do_not_inherit):
+ setattr(self, k, copy.deepcopy(getattr(other, k)))
+
+ msg = 'inherit from %s' % (repr(other),)
+ self.inheritance_path = other.inheritance_path + (msg,)
+
+ def copy(self):
+ job = Job(self.name)
+ for k in self.attributes:
+ if self._get(k) is not None:
+ setattr(job, k, copy.deepcopy(self._get(k)))
+ return job
+
+ def applyVariant(self, other):
+ """Copy the attributes which have been set on the other job to this
+ job."""
if not isinstance(other, Job):
raise Exception("Job unable to inherit from %s" % (other,))
- self.inheritance_path.extend(other.inheritance_path)
- self.inheritance_path.append('%s %s' % (repr(other), comment))
- for k, v in self.attributes.items():
- if (getattr(other, k) != v and k not in
- set(['auth', 'pre_run', 'post_run', 'inheritance_path'])):
- setattr(self, k, getattr(other, k))
- # Inherit auth only if explicitly allowed
- if other.auth and 'inherit' in other.auth and other.auth['inherit']:
- setattr(self, 'auth', getattr(other, 'auth'))
- # Pre and post run are lists; make a copy
- self.pre_run = other.pre_run + self.pre_run
- self.post_run = self.post_run + other.post_run
+
+ for k in self.execution_attributes:
+ if (other._get(k) is not None and
+ k not in set(['final'])):
+ if self.final:
+ raise Exception("Unable to modify final job %s attribute "
+ "%s=%s with variant %s" % (
+ repr(self), k, other._get(k),
+ repr(other)))
+ if k not in set(['pre_run', 'post_run']):
+ setattr(self, k, copy.deepcopy(other._get(k)))
+
+ # Don't set final above so that we don't trip an error halfway
+ # through assignment.
+ if other.final != self.attributes['final']:
+ self.final = other.final
+
+ if other._get('pre_run') is not None:
+ self.pre_run = self.pre_run + other.pre_run
+ if other._get('post_run') is not None:
+ self.post_run = other.post_run + self.post_run
+
+ for k in self.context_attributes:
+ if (other._get(k) is not None and
+ k not in set(['tags'])):
+ setattr(self, k, copy.deepcopy(other._get(k)))
+
+ if other._get('tags') is not None:
+ self.tags = self.tags.union(other.tags)
+
+ msg = 'apply variant %s' % (repr(other),)
+ self.inheritance_path = self.inheritance_path + (msg,)
def changeMatches(self, change):
if self.branch_matcher and not self.branch_matcher.matches(change):
@@ -723,16 +815,18 @@
return ret
return None
- def inheritFrom(self, other, comment='unknown'):
+ def inheritFrom(self, other):
if other.job:
- self.job = Job(other.job.name)
- self.job.inheritFrom(other.job, comment)
+ if not self.job:
+ self.job = other.job.copy()
+ else:
+ self.job.applyVariant(other.job)
for other_tree in other.job_trees:
this_tree = self.getJobTreeForJob(other_tree.job)
if not this_tree:
this_tree = JobTree(None)
self.job_trees.append(this_tree)
- this_tree.inheritFrom(other_tree, comment)
+ this_tree.inheritFrom(other_tree)
class Build(object):
@@ -1997,25 +2091,28 @@
job = tree.job
if not job.changeMatches(change):
continue
- frozen_job = Job(job.name)
- frozen_tree = JobTree(frozen_job)
- inherited = set()
+ frozen_job = None
+ matched = False
for variant in self.getJobs(job.name):
if variant.changeMatches(change):
- if variant not in inherited:
- frozen_job.inheritFrom(variant,
- 'variant while freezing')
- inherited.add(variant)
- if not inherited:
+ if frozen_job is None:
+ frozen_job = variant.copy()
+ frozen_job.setRun()
+ else:
+ frozen_job.applyVariant(variant)
+ matched = True
+ if not matched:
# A change must match at least one defined job variant
# (that is to say that it must match more than just
# the job that is defined in the tree).
continue
- if job not in inherited:
- # Only update from the job in the tree if it is
- # unique, otherwise we might unset an attribute we
- # have overloaded.
- frozen_job.inheritFrom(job, 'tree job while freezing')
+ # If the job does not allow auth inheritance, do not allow
+ # the project-pipeline variant to update its execution
+ # attributes.
+ if frozen_job.auth and not frozen_job.auth.get('inherit'):
+ frozen_job.final = True
+ frozen_job.applyVariant(job)
+ frozen_tree = JobTree(frozen_job)
parent.job_trees.append(frozen_tree)
self._createJobTree(change, tree.job_trees, frozen_tree)
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 38187cf..c042e4f 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -620,8 +620,6 @@
change = pipeline.source.getChange(event, project)
self.log.debug("Event %s for change %s was directly assigned "
"to pipeline %s" % (event, change, self))
- self.log.info("Adding %s %s to %s" %
- (project, change, pipeline))
pipeline.manager.addChange(change, ignore_requirements=True)
def _areAllBuildsComplete(self):
@@ -727,8 +725,6 @@
elif event.type == 'change-abandoned':
pipeline.manager.removeAbandonedChange(change)
if pipeline.manager.eventMatches(event, change):
- self.log.info("Adding %s %s to %s" %
- (change.project, change, pipeline))
pipeline.manager.addChange(change)
finally:
self.trigger_event_queue.task_done()