Merge "Re-enable test_json_status" into feature/zuulv3
diff --git a/.testr.conf b/.testr.conf
index e8f41cd..7e8d028 100644
--- a/.testr.conf
+++ b/.testr.conf
@@ -1,4 +1,4 @@
[DEFAULT]
-test_command=OS_LOG_LEVEL=${OS_LOG_LEVEL:-INFO} OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} OS_LOG_DEFAULTS=${OS_LOG_DEFAULTS:-""} ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./tests/unit} $LISTOPT $IDOPTION
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} OS_LOG_DEFAULTS=${OS_LOG_DEFAULTS:-""} ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./tests/unit} $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
diff --git a/tests/base.py b/tests/base.py
index 1b65416..506e22c 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -28,10 +28,12 @@
import select
import shutil
from six.moves import reload_module
+from six import StringIO
import socket
import string
import subprocess
import swiftclient
+import sys
import tempfile
import threading
import time
@@ -43,6 +45,8 @@
import kazoo.exceptions
import statsd
import testtools
+import testtools.content
+import testtools.content_type
from git.exc import NoSuchPathError
import zuul.driver.gerrit.gerritsource as gerritsource
@@ -64,10 +68,6 @@
'fixtures')
USE_TEMPDIR = True
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
def repack_repo(path):
cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
@@ -557,13 +557,14 @@
if len(self.parameters.get('nodes')) == 1:
self.node = self.parameters['nodes'][0]['image']
self.unique = self.parameters['ZUUL_UUID']
+ self.pipeline = self.parameters['ZUUL_PIPELINE']
+ self.project = self.parameters['ZUUL_PROJECT']
self.name = self.parameters['job']
self.wait_condition = threading.Condition()
self.waiting = False
self.aborted = False
self.requeue = False
self.created = time.time()
- self.run_error = False
self.changes = None
if 'ZUUL_CHANGE_IDS' in self.parameters:
self.changes = self.parameters['ZUUL_CHANGE_IDS']
@@ -572,7 +573,8 @@
waiting = ''
if self.waiting:
waiting = ' [waiting]'
- return '<FakeBuild %s %s%s>' % (self.name, self.changes, waiting)
+ return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
+ self.changes, waiting)
def release(self):
"""Release this build."""
@@ -612,16 +614,13 @@
self._wait()
self.log.debug("Build %s continuing" % self.unique)
- result = 'SUCCESS'
+ result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
- result = 'FAILURE'
+ result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
if self.aborted:
- result = 'ABORTED'
+ result = (RecordingAnsibleJob.RESULT_ABORTED, None)
if self.requeue:
- result = None
-
- if self.run_error:
- result = 'RUN_ERROR'
+ result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
return result
@@ -676,6 +675,7 @@
"""
def __init__(self, *args, **kw):
self._run_ansible = kw.pop('_run_ansible', False)
+ self._test_root = kw.pop('_test_root', False)
super(RecordingLaunchServer, self).__init__(*args, **kw)
self.hold_jobs_in_build = False
self.lock = threading.Lock()
@@ -724,7 +724,11 @@
job.build = build
self.running_builds.append(build)
self.job_builds[job.unique] = build
- super(RecordingLaunchServer, self).launchJob(job)
+ args = json.loads(job.arguments)
+ args['zuul']['_test'] = dict(test_root=self._test_root)
+ job.arguments = json.dumps(args)
+ self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
+ self.job_workers[job.unique].run()
def stopJob(self, job):
self.log.debug("handle stop")
@@ -736,27 +740,33 @@
build.release()
super(RecordingLaunchServer, self).stopJob(job)
- def runAnsible(self, jobdir, job):
- build = self.job_builds[job.unique]
- build.jobdir = jobdir
- if self._run_ansible:
- result = super(RecordingLaunchServer, self).runAnsible(jobdir, job)
- else:
- result = build.run()
+class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
+ def runPlaybooks(self):
+ build = self.launcher_server.job_builds[self.job.unique]
+ build.jobdir = self.jobdir
- self.lock.acquire()
- self.build_history.append(
+ result = super(RecordingAnsibleJob, self).runPlaybooks()
+
+ self.launcher_server.lock.acquire()
+ self.launcher_server.build_history.append(
BuildHistory(name=build.name, result=result, changes=build.changes,
node=build.node, uuid=build.unique,
parameters=build.parameters,
pipeline=build.parameters['ZUUL_PIPELINE'])
)
- self.running_builds.remove(build)
- del self.job_builds[job.unique]
- self.lock.release()
- if build.run_error:
- result = None
+ self.launcher_server.running_builds.remove(build)
+ del self.launcher_server.job_builds[self.job.unique]
+ self.launcher_server.lock.release()
+ return result
+
+ def runAnsible(self, cmd, timeout):
+ build = self.launcher_server.job_builds[self.job.unique]
+
+ if self.launcher_server._run_ansible:
+ result = super(RecordingAnsibleJob, self).runAnsible(cmd, timeout)
+ else:
+ result = build.run()
return result
@@ -908,10 +918,13 @@
reqs = []
for oid in sorted(reqids):
path = self.REQUEST_ROOT + '/' + oid
- data, stat = self.client.get(path)
- data = json.loads(data)
- data['_oid'] = oid
- reqs.append(data)
+ try:
+ data, stat = self.client.get(path)
+ data = json.loads(data)
+ data['_oid'] = oid
+ reqs.append(data)
+ except kazoo.exceptions.NoNodeError:
+ pass
return reqs
def getNodes(self):
@@ -1041,6 +1054,20 @@
class BaseTestCase(testtools.TestCase):
log = logging.getLogger("zuul.test")
+ def attachLogs(self, *args):
+ def reader():
+ self._log_stream.seek(0)
+ while True:
+ x = self._log_stream.read(4096)
+ if not x:
+ break
+ yield x.encode('utf8')
+ content = testtools.content.content_from_reader(
+ reader,
+ testtools.content_type.UTF8_TEXT,
+ False)
+ self.addDetail('logging', content)
+
def setUp(self):
super(BaseTestCase, self).setUp()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
@@ -1062,40 +1089,37 @@
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
os.environ.get('OS_LOG_CAPTURE') == '1'):
- log_level = logging.DEBUG
- if os.environ.get('OS_LOG_LEVEL') == 'DEBUG':
- log_level = logging.DEBUG
- elif os.environ.get('OS_LOG_LEVEL') == 'INFO':
- log_level = logging.INFO
- elif os.environ.get('OS_LOG_LEVEL') == 'WARNING':
- log_level = logging.WARNING
- elif os.environ.get('OS_LOG_LEVEL') == 'ERROR':
- log_level = logging.ERROR
- elif os.environ.get('OS_LOG_LEVEL') == 'CRITICAL':
- log_level = logging.CRITICAL
- self.useFixture(fixtures.FakeLogger(
- level=log_level,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s'))
+ self._log_stream = StringIO()
+ self.addOnException(self.attachLogs)
+ else:
+ self._log_stream = sys.stdout
- # NOTE(notmorgan): Extract logging overrides for specific libraries
- # from the OS_LOG_DEFAULTS env and create FakeLogger fixtures for
- # each. This is used to limit the output during test runs from
- # libraries that zuul depends on such as gear.
+ handler = logging.StreamHandler(self._log_stream)
+ formatter = logging.Formatter('%(asctime)s %(name)-32s '
+ '%(levelname)-8s %(message)s')
+ handler.setFormatter(formatter)
+
+ logger = logging.getLogger()
+ logger.setLevel(logging.DEBUG)
+ logger.addHandler(handler)
+
+ # NOTE(notmorgan): Extract logging overrides for specific
+ # libraries from the OS_LOG_DEFAULTS env and create loggers
+ # for each. This is used to limit the output during test runs
+ # from libraries that zuul depends on such as gear.
log_defaults_from_env = os.environ.get(
'OS_LOG_DEFAULTS',
- 'git.cmd=INFO,kazoo.client=INFO')
+ 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
if log_defaults_from_env:
for default in log_defaults_from_env.split(','):
try:
name, level_str = default.split('=', 1)
level = getattr(logging, level_str, logging.DEBUG)
- self.useFixture(fixtures.FakeLogger(
- name=name,
- level=level,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s'))
+ logger = logging.getLogger(name)
+ logger.setLevel(level)
+ logger.addHandler(handler)
+ logger.propagate = False
except ValueError:
# NOTE(notmorgan): Invalid format of the log default,
# skip and don't try and apply a logger for the
@@ -1218,6 +1242,8 @@
self.gearman_server = FakeGearmanServer()
self.config.set('gearman', 'port', str(self.gearman_server.port))
+ self.log.info("Gearman server on port %s" %
+ (self.gearman_server.port,))
gerritsource.GerritSource.replication_timeout = 1.5
gerritsource.GerritSource.replication_retry_interval = 0.5
@@ -1249,7 +1275,10 @@
self._startMerger()
self.launch_server = RecordingLaunchServer(
- self.config, self.connections, _run_ansible=self.run_ansible)
+ self.config, self.connections,
+ jobdir_root=self.test_root,
+ _run_ansible=self.run_ansible,
+ _test_root=self.test_root)
self.launch_server.start()
self.history = self.launch_server.build_history
self.builds = self.launch_server.running_builds
@@ -1559,7 +1588,7 @@
self.log.debug("Waiting until settled...")
start = time.time()
while True:
- if time.time() - start > 10:
+ if time.time() - start > 20:
self.log.error("Timeout waiting for Zuul to settle")
self.log.error("Queue status:")
for queue in self.event_queues:
@@ -1583,10 +1612,15 @@
self.eventQueuesJoin()
self.sched.run_handler_lock.acquire()
if (not self.merge_client.jobs and
- all(self.eventQueuesEmpty()) and
self.haveAllBuildsReported() and
self.areAllBuildsWaiting() and
- self.areAllNodeRequestsComplete()):
+ self.areAllNodeRequestsComplete() and
+ all(self.eventQueuesEmpty())):
+ # The queue empty check is placed at the end to
+ # ensure that if a component adds an event between
+ # when locked the run handler and checked that the
+ # components were stable, we don't erroneously
+ # report that we are settled.
self.sched.run_handler_lock.release()
self.launch_server.lock.release()
self.log.debug("...settled.")
@@ -1749,6 +1783,10 @@
zuul.merger.merger.reset_repo_to_head(repo)
for fn, content in files.items():
fn = os.path.join(path, fn)
+ try:
+ os.makedirs(os.path.dirname(fn))
+ except OSError:
+ pass
with open(fn, 'w') as f:
f.write(content)
repo.index.add([fn])
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/post.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/post.yaml
new file mode 100644
index 0000000..2e512b1
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/post.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+ tasks:
+ - file:
+ path: "{{zuul._test.test_root}}/{{zuul.uuid}}.post.flag"
+ state: touch
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/pre.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/pre.yaml
new file mode 100644
index 0000000..f4222ff
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/pre.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+ tasks:
+ - file:
+ path: "{{zuul._test.test_root}}/{{zuul.uuid}}.pre.flag"
+ state: touch
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
new file mode 100644
index 0000000..6b0af99
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+ tasks:
+ - file:
+ path: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
+ state: touch
diff --git a/tests/fixtures/config/merge-modes/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
similarity index 63%
copy from tests/fixtures/config/merge-modes/git/common-config/zuul.yaml
copy to tests/fixtures/config/ansible/git/common-config/zuul.yaml
index a7a4c78..7964243 100644
--- a/tests/fixtures/config/merge-modes/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -37,26 +37,6 @@
precedence: high
- job:
- name:
- project-test1
-
-- project:
- name: org/project-merge
- merge-mode: merge
- gate:
- jobs:
- - project-test1
-
-- project:
- name: org/project-merge-resolve
- merge-mode: merge-resolve
- gate:
- jobs:
- - project-test1
-
-- project:
- name: org/project-cherry-pick
- merge-mode: cherry-pick
- gate:
- jobs:
- - project-test1
+ name: python27
+ pre-run: pre
+ post-run: post
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..6bedb07
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -0,0 +1,6 @@
+- project:
+ name: org/project
+
+ check:
+ jobs:
+ - python27
diff --git a/tests/fixtures/config/merge-modes/git/org_project-merge/README b/tests/fixtures/config/ansible/git/org_project/README
similarity index 100%
copy from tests/fixtures/config/merge-modes/git/org_project-merge/README
copy to tests/fixtures/config/ansible/git/org_project/README
diff --git a/tests/fixtures/config/ansible/main.yaml b/tests/fixtures/config/ansible/main.yaml
new file mode 100644
index 0000000..d9868fa
--- /dev/null
+++ b/tests/fixtures/config/ansible/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-repos:
+ - common-config
+ project-repos:
+ - org/project
diff --git a/tests/fixtures/config/duplicate-pipeline/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/duplicate-pipeline/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/duplicate-pipeline/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/in-repo/git/org_project/playbooks/project-test1.yaml b/tests/fixtures/config/in-repo/git/org_project/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/in-repo/git/org_project/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/merge-modes/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/merge-modes/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/merge-modes/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/merges/git/common-config/playbooks/project-merge.yaml b/tests/fixtures/config/merges/git/common-config/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/merges/git/common-config/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/merges/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/merges/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/merges/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/merges/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/merges/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/merges/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/merge-modes/git/common-config/zuul.yaml b/tests/fixtures/config/merges/git/common-config/zuul.yaml
similarity index 78%
rename from tests/fixtures/config/merge-modes/git/common-config/zuul.yaml
rename to tests/fixtures/config/merges/git/common-config/zuul.yaml
index a7a4c78..bb91f3a 100644
--- a/tests/fixtures/config/merge-modes/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/merges/git/common-config/zuul.yaml
@@ -40,6 +40,15 @@
name:
project-test1
+- job:
+ name:
+ project-test2
+
+- job:
+ name:
+ project-merge
+ hold-following-changes: true
+
- project:
name: org/project-merge
merge-mode: merge
@@ -60,3 +69,12 @@
gate:
jobs:
- project-test1
+
+- project:
+ name: org/project-merge-branches
+ merge-mode: cherry-pick
+ gate:
+ jobs:
+ - project-merge:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/config/merge-modes/git/org_project-cherry-pick/README b/tests/fixtures/config/merges/git/org_project-cherry-pick/README
similarity index 100%
rename from tests/fixtures/config/merge-modes/git/org_project-cherry-pick/README
rename to tests/fixtures/config/merges/git/org_project-cherry-pick/README
diff --git a/tests/fixtures/config/merge-modes/git/org_project-cherry-pick/README b/tests/fixtures/config/merges/git/org_project-merge-branches/README
similarity index 100%
copy from tests/fixtures/config/merge-modes/git/org_project-cherry-pick/README
copy to tests/fixtures/config/merges/git/org_project-merge-branches/README
diff --git a/tests/fixtures/config/merge-modes/git/org_project-merge-resolve/README b/tests/fixtures/config/merges/git/org_project-merge-resolve/README
similarity index 100%
rename from tests/fixtures/config/merge-modes/git/org_project-merge-resolve/README
rename to tests/fixtures/config/merges/git/org_project-merge-resolve/README
diff --git a/tests/fixtures/config/merge-modes/git/org_project-merge/README b/tests/fixtures/config/merges/git/org_project-merge/README
similarity index 100%
rename from tests/fixtures/config/merge-modes/git/org_project-merge/README
rename to tests/fixtures/config/merges/git/org_project-merge/README
diff --git a/tests/fixtures/config/merge-modes/main.yaml b/tests/fixtures/config/merges/main.yaml
similarity index 100%
rename from tests/fixtures/config/merge-modes/main.yaml
rename to tests/fixtures/config/merges/main.yaml
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/multi-tenant/git/common-config/playbooks/python27.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/common-config/playbooks/python27.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/playbooks/project1-test1.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/playbooks/project1-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/playbooks/project1-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/playbooks/project2-test1.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/playbooks/project2-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/playbooks/project2-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/one-job-project/git/common-config/playbooks/one-job-project-merge.yaml b/tests/fixtures/config/one-job-project/git/common-config/playbooks/one-job-project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/one-job-project/git/common-config/playbooks/one-job-project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/one-job-project/git/common-config/playbooks/one-job-project-post.yaml b/tests/fixtures/config/one-job-project/git/common-config/playbooks/one-job-project-post.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/one-job-project/git/common-config/playbooks/one-job-project-post.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/base.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/python27.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/python27.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/playbooks/python27.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/python35.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/python35.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/playbooks/python35.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/email/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/email/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/email/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/email/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/email/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/email/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/newer-than/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/newer-than/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/newer-than/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/newer-than/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/newer-than/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/newer-than/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/older-than/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/older-than/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/older-than/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/older-than/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/older-than/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/older-than/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/reject-username/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/reject-username/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/reject-username/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/reject-username/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/reject-username/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/reject-username/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/reject/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/reject/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/reject/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/reject/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/reject/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/reject/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/state/git/common-config/playbooks/project-job.yaml b/tests/fixtures/config/requirements/state/git/common-config/playbooks/project-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/state/git/common-config/playbooks/project-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/username/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/username/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/username/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/username/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/username/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/username/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/vote1/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/vote1/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/vote1/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/vote1/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/vote1/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/vote1/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/vote2/git/common-config/playbooks/project1-job.yaml b/tests/fixtures/config/requirements/vote2/git/common-config/playbooks/project1-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/vote2/git/common-config/playbooks/project1-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/requirements/vote2/git/common-config/playbooks/project2-job.yaml b/tests/fixtures/config/requirements/vote2/git/common-config/playbooks/project2-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/requirements/vote2/git/common-config/playbooks/project2-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/experimental-project-test.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/experimental-project-test.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/experimental-project-test.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-merge.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-test1.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-test2.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/nonvoting-project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-merge.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-post.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-post.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-post.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-testfile.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-testfile.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project-testfile.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/playbooks/project1-project2-integration.yaml b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project1-project2-integration.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/common-config/playbooks/project1-project2-integration.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index ad7c352..b91bf6f 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -123,6 +123,7 @@
jobs:
- project-test1
- project-test2
+ - project-testfile
post:
jobs:
- project-post
diff --git a/tests/fixtures/config/single-tenant/git/layout-disabled-at/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-disabled-at/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-disabled-at/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/playbooks/project-post.yaml b/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/playbooks/project-post.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/playbooks/project-post.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-irrelevant-starts-empty.yaml b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-irrelevant-starts-empty.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-irrelevant-starts-empty.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-irrelevant-starts-full.yaml b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-irrelevant-starts-full.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-irrelevant-starts-full.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-nomatch-starts-empty.yaml b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-nomatch-starts-empty.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-nomatch-starts-empty.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-nomatch-starts-full.yaml b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-nomatch-starts-full.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-inheritance/playbooks/project-test-nomatch-starts-full.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/playbooks/project-test-irrelevant-files.yaml b/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/playbooks/project-test-irrelevant-files.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/playbooks/project-test-irrelevant-files.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/mutex-one.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/mutex-one.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/mutex-one.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/mutex-two.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/mutex-two.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/mutex-two.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-mutex/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml
new file mode 100644
index 0000000..e91903a
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml
@@ -0,0 +1,33 @@
+- pipeline:
+ name: check
+ manager: independent
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: project-test1
+
+- job:
+ name: mutex-one
+ mutex: test-mutex
+
+- job:
+ name: mutex-two
+ mutex: test-mutex
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ - mutex-one
+ - mutex-two
diff --git a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-merge.yaml b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-test2.yaml b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/experimental-project-test.yaml b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/experimental-project-test.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/experimental-project-test.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-merge.yaml b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-test2.yaml b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-smtp/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/success-url/git/common-config/playbooks/docs-draft-test.yaml b/tests/fixtures/config/success-url/git/common-config/playbooks/docs-draft-test.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/success-url/git/common-config/playbooks/docs-draft-test.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/success-url/git/common-config/playbooks/docs-draft-test2.yaml b/tests/fixtures/config/success-url/git/common-config/playbooks/docs-draft-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/success-url/git/common-config/playbooks/docs-draft-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-foo-test5.yaml b/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-foo-test5.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-foo-test5.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-test3.yaml b/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-test3.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-test3.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-test4.yaml b/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-test4.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/playbooks/layered-project-test4.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test6.yaml b/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test6.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/playbooks/project-test6.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
new file mode 100644
index 0000000..302dfcf
--- /dev/null
+++ b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
@@ -0,0 +1,42 @@
+- pipeline:
+ name: review_check
+ manager: independent
+ source: review_gerrit
+ trigger:
+ review_gerrit:
+ - event: patchset-created
+ success:
+ review_gerrit:
+ verified: 1
+ failure:
+ review_gerrit:
+ verified: -1
+
+- pipeline:
+ name: another_check
+ manager: independent
+ source: another_gerrit
+ trigger:
+ another_gerrit:
+ - event: patchset-created
+ success:
+ another_gerrit:
+ verified: 1
+ failure:
+ another_gerrit:
+ verified: -1
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- project:
+ name: org/project1
+ review_check:
+ jobs:
+ - project-test1
+ another_check:
+ jobs:
+ - project-test2
diff --git a/tests/fixtures/config/merge-modes/git/org_project-cherry-pick/README b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/org_project1/README
similarity index 100%
copy from tests/fixtures/config/merge-modes/git/org_project-cherry-pick/README
copy to tests/fixtures/config/zuul-connections-multiple-gerrits/git/org_project1/README
diff --git a/tests/fixtures/config/zuul-connections-multiple-gerrits/main.yaml b/tests/fixtures/config/zuul-connections-multiple-gerrits/main.yaml
new file mode 100644
index 0000000..730cc7e
--- /dev/null
+++ b/tests/fixtures/config/zuul-connections-multiple-gerrits/main.yaml
@@ -0,0 +1,6 @@
+- tenant:
+ name: tenant-one
+ source:
+ review_gerrit:
+ config-repos:
+ - common-config
diff --git a/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/playbooks/project-check.yaml b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/playbooks/project-check.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/playbooks/project-check.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/playbooks/project-gate.yaml b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/playbooks/project-gate.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/playbooks/project-gate.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/playbooks/project-check.yaml b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/playbooks/project-check.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/playbooks/project-check.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/playbooks/project-gate.yaml b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/playbooks/project-gate.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/playbooks/project-gate.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/layout-connections-multiple-gerrits.yaml b/tests/fixtures/layout-connections-multiple-gerrits.yaml
deleted file mode 100644
index 029f42f..0000000
--- a/tests/fixtures/layout-connections-multiple-gerrits.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- source: review_gerrit
- trigger:
- review_gerrit:
- - event: patchset-created
- success:
- review_gerrit:
- VRFY: 1
- failure:
- review_gerrit:
- VRFY: -1
-
- - name: another_check
- manager: IndependentPipelineManager
- source: another_gerrit
- trigger:
- another_gerrit:
- - event: patchset-created
- success:
- another_gerrit:
- VRFY: 1
- failure:
- another_gerrit:
- VRFY: -1
-
-projects:
- - name: org/project
- check:
- - project-review-gerrit
- another_check:
- - project-another-gerrit
-
- - name: org/project1
- another_check:
- - project1-another-gerrit
diff --git a/tests/fixtures/layout-mutex.yaml b/tests/fixtures/layout-mutex.yaml
deleted file mode 100644
index fcd0529..0000000
--- a/tests/fixtures/layout-mutex.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
-jobs:
- - name: mutex-one
- mutex: test-mutex
- - name: mutex-two
- mutex: test-mutex
-
-projects:
- - name: org/project
- check:
- - project-test1
- - mutex-one
- - mutex-two
diff --git a/tests/fixtures/zuul-connections-multiple-gerrits.conf b/tests/fixtures/zuul-connections-multiple-gerrits.conf
index f067e6e..89f0aa6 100644
--- a/tests/fixtures/zuul-connections-multiple-gerrits.conf
+++ b/tests/fixtures/zuul-connections-multiple-gerrits.conf
@@ -2,7 +2,7 @@
server=127.0.0.1
[zuul]
-layout_config=layout-connections-multiple-voters.yaml
+tenant_config=main.yaml
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
job_name_in_report=true
diff --git a/tests/make_playbooks.py b/tests/make_playbooks.py
new file mode 100755
index 0000000..12d9e71
--- /dev/null
+++ b/tests/make_playbooks.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import yaml
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+ 'fixtures')
+CONFIG_DIR = os.path.join(FIXTURE_DIR, 'config')
+
+
+def make_playbook(path):
+ d = os.path.dirname(path)
+ try:
+ os.makedirs(d)
+ except OSError:
+ pass
+ with open(path, 'w') as f:
+ f.write('- hosts: all\n')
+ f.write(' tasks: []\n')
+
+
+def handle_repo(path):
+ print('Repo: %s' % path)
+ config_path = None
+ for fn in ['zuul.yaml', '.zuul.yaml']:
+ if os.path.exists(os.path.join(path, fn)):
+ config_path = os.path.join(path, fn)
+ break
+ config = yaml.load(open(config_path))
+ for block in config:
+ if 'job' not in block:
+ continue
+ job = block['job']['name']
+ playbook = os.path.join(path, 'playbooks', job + '.yaml')
+ if not os.path.exists(playbook):
+ print(' Creating: %s' % job)
+ make_playbook(playbook)
+
+
+def main():
+ repo_dirs = []
+
+ for root, dirs, files in os.walk(CONFIG_DIR):
+ if 'zuul.yaml' in files or '.zuul.yaml' in files:
+ repo_dirs.append(root)
+
+ for path in repo_dirs:
+ handle_repo(path)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/nodepool/test_nodepool_integration.py b/tests/nodepool/test_nodepool_integration.py
index 881aae7..ef459e4 100644
--- a/tests/nodepool/test_nodepool_integration.py
+++ b/tests/nodepool/test_nodepool_integration.py
@@ -53,8 +53,7 @@
# Test a simple node request
nodeset = model.NodeSet()
- nodeset.addNode(model.Node('controller', 'fake-label'))
- nodeset.addNode(model.Node('compute', 'fake-label'))
+ nodeset.addNode(model.Node('controller', 'fake-nodepool'))
job = model.Job('testjob')
job.nodeset = nodeset
request = self.nodepool.requestNodes(None, job)
diff --git a/tests/print_layout.py b/tests/print_layout.py
index 9afd379..a295886 100644
--- a/tests/print_layout.py
+++ b/tests/print_layout.py
@@ -16,9 +16,9 @@
import os
import sys
-import tests.base
-
-CONFIG_DIR = os.path.join(tests.base.FIXTURE_DIR, 'config')
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+ 'fixtures')
+CONFIG_DIR = os.path.join(FIXTURE_DIR, 'config')
def print_file(title, path):
diff --git a/tests/unit/test_clonemapper.py b/tests/unit/test_clonemapper.py
index b7814f8..bd8c8b0 100644
--- a/tests/unit/test_clonemapper.py
+++ b/tests/unit/test_clonemapper.py
@@ -13,14 +13,9 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import testtools
from zuul.lib.clonemapper import CloneMapper
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-17s '
- '%(levelname)-8s %(message)s')
-
class TestCloneMapper(testtools.TestCase):
diff --git a/tests/unit/test_cloner.py b/tests/unit/test_cloner.py
index 67b5303..2cdc826 100644
--- a/tests/unit/test_cloner.py
+++ b/tests/unit/test_cloner.py
@@ -26,10 +26,6 @@
from tests.base import ZuulTestCase
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestCloner(ZuulTestCase):
diff --git a/tests/unit/test_cloner_cmd.py b/tests/unit/test_cloner_cmd.py
index 9cbb5b8..2d8747f 100644
--- a/tests/unit/test_cloner_cmd.py
+++ b/tests/unit/test_cloner_cmd.py
@@ -12,16 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import os
import testtools
import zuul.cmd.cloner
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestClonerCmdArguments(testtools.TestCase):
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index f8d1bf5..d9bc72f 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -47,28 +47,48 @@
class TestMultipleGerrits(ZuulTestCase):
- def setUp(self):
- self.skip("Disabled for early v3 development")
- def setup_config(self,
- config_file='zuul-connections-multiple-gerrits.conf'):
- super(TestMultipleGerrits, self).setup_config(config_file)
- self.self.updateConfigLayout(
- 'layout-connections-multiple-gerrits.yaml')
+ config_file = 'zuul-connections-multiple-gerrits.conf'
+ tenant_config_file = 'config/zuul-connections-multiple-gerrits/main.yaml'
def test_multiple_project_separate_gerrits(self):
- self.worker.hold_jobs_in_build = True
+ self.launch_server.hold_jobs_in_build = True
A = self.fake_another_gerrit.addFakeChange(
- 'org/project', 'master', 'A')
+ 'org/project1', 'master', 'A')
self.fake_another_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
- self.assertEqual(1, len(self.builds))
- self.assertEqual('project-another-gerrit', self.builds[0].name)
- self.assertTrue(self.job_has_changes(self.builds[0], A))
+ self.assertBuilds([dict(name='project-test2',
+ changes='1,1',
+ project='org/project1',
+ pipeline='another_check')])
- self.worker.hold_jobs_in_build = False
- self.worker.release()
+ # NOTE(jamielennox): the tests back the git repo for both connections
+ # onto the same git repo on the file system. If we just create another
+ # fake change the fake_review_gerrit will try to create another 1,1
+ # change and git will fail to create the ref. Arbitrarily set it to get
+ # around the problem.
+ self.fake_review_gerrit.change_number = 50
+
+ B = self.fake_review_gerrit.addFakeChange(
+ 'org/project1', 'master', 'B')
+ self.fake_review_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+
+ self.waitUntilSettled()
+
+ self.assertBuilds([
+ dict(name='project-test2',
+ changes='1,1',
+ project='org/project1',
+ pipeline='another_check'),
+ dict(name='project-test1',
+ changes='51,1',
+ project='org/project1',
+ pipeline='review_check'),
+ ])
+
+ self.launch_server.hold_jobs_in_build = False
+ self.launch_server.release()
self.waitUntilSettled()
diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py
index 5062c14..f815344 100644
--- a/tests/unit/test_merger_repo.py
+++ b/tests/unit/test_merger_repo.py
@@ -23,10 +23,6 @@
from zuul.merger.merger import Repo
from tests.base import ZuulTestCase
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestMergerRepo(ZuulTestCase):
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 0189340..b7dc706 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -30,7 +30,10 @@
@property
def job(self):
layout = model.Layout()
+ project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
job = configloader.JobParser.fromYaml(layout, {
+ '_source_context': context,
'name': 'job',
'irrelevant-files': [
'^docs/.*$'
@@ -56,32 +59,46 @@
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
- project = model.Project('project')
+ project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
+ 'nodes': [{
+ 'name': 'controller',
+ 'image': 'base',
+ }],
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'parent': 'base',
+ 'nodes': [{
+ 'name': 'controller',
+ 'image': 'new',
+ }],
'timeout': 40,
})
layout.addJob(python27)
python27diablo = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'branches': [
'stable/diablo'
],
+ 'nodes': [{
+ 'name': 'controller',
+ 'image': 'old',
+ }],
'timeout': 50,
})
layout.addJob(python27diablo)
project_config = configloader.ProjectParser.fromYaml(layout, {
+ '_source_context': context,
'name': 'project',
'gate': {
'jobs': [
@@ -92,6 +109,7 @@
layout.addProjectConfig(project_config, update_pipeline=False)
change = model.Change(project)
+ # Test master
change.branch = 'master'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
@@ -105,7 +123,11 @@
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
self.assertEqual(job.timeout, 40)
+ nodes = job.nodeset.getNodes()
+ self.assertEqual(len(nodes), 1)
+ self.assertEqual(nodes[0].image, 'new')
+ # Test diablo
change.branch = 'stable/diablo'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
@@ -119,19 +141,23 @@
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
self.assertEqual(job.timeout, 50)
+ nodes = job.nodeset.getNodes()
+ self.assertEqual(len(nodes), 1)
+ self.assertEqual(nodes[0].image, 'old')
def test_job_auth_inheritance(self):
layout = model.Layout()
- project = model.Project('project')
+ project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
})
layout.addJob(base)
pypi_upload_without_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'pypi-upload-without-inherit',
'parent': 'base',
'timeout': 40,
@@ -143,7 +169,7 @@
})
layout.addJob(pypi_upload_without_inherit)
pypi_upload_with_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'pypi-upload-with-inherit',
'parent': 'base',
'timeout': 40,
@@ -157,7 +183,7 @@
layout.addJob(pypi_upload_with_inherit)
pypi_upload_with_inherit_false = configloader.JobParser.fromYaml(
layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'pypi-upload-with-inherit-false',
'parent': 'base',
'timeout': 40,
@@ -170,20 +196,20 @@
})
layout.addJob(pypi_upload_with_inherit_false)
in_repo_job_without_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'in-repo-job-without-inherit',
'parent': 'pypi-upload-without-inherit',
})
layout.addJob(in_repo_job_without_inherit)
in_repo_job_with_inherit = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'in-repo-job-with-inherit',
'parent': 'pypi-upload-with-inherit',
})
layout.addJob(in_repo_job_with_inherit)
in_repo_job_with_inherit_false = configloader.JobParser.fromYaml(
layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'in-repo-job-with-inherit-false',
'parent': 'pypi-upload-with-inherit-false',
})
@@ -201,23 +227,24 @@
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
- project = model.Project('project')
+ project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'parent': 'base',
'timeout': 40,
})
layout.addJob(python27)
python27diablo = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'branches': [
'stable/diablo'
@@ -227,6 +254,7 @@
layout.addJob(python27diablo)
project_config = configloader.ProjectParser.fromYaml(layout, {
+ '_source_context': context,
'name': 'project',
'gate': {
'jobs': [
@@ -271,16 +299,17 @@
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
- project = model.Project('project')
+ project = model.Project('project', None)
+ context = model.SourceContext(project, 'master', True)
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'base',
'timeout': 30,
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(layout, {
- '_source_project': project,
+ '_source_context': context,
'name': 'python27',
'parent': 'base',
'timeout': 40,
@@ -289,6 +318,7 @@
layout.addJob(python27)
project_config = configloader.ProjectParser.fromYaml(layout, {
+ '_source_context': context,
'name': 'project',
'gate': {
'jobs': [
@@ -312,16 +342,19 @@
def test_job_source_project(self):
layout = model.Layout()
- base_project = model.Project('base_project')
+ base_project = model.Project('base_project', None)
+ base_context = model.SourceContext(base_project, 'master', True)
+
base = configloader.JobParser.fromYaml(layout, {
- '_source_project': base_project,
+ '_source_context': base_context,
'name': 'base',
})
layout.addJob(base)
- other_project = model.Project('other_project')
+ other_project = model.Project('other_project', None)
+ other_context = model.SourceContext(other_project, 'master', True)
base2 = configloader.JobParser.fromYaml(layout, {
- '_source_project': other_project,
+ '_source_context': other_context,
'name': 'base',
})
with testtools.ExpectedException(
diff --git a/tests/unit/test_openstack.py b/tests/unit/test_openstack.py
index 175b4bd..d0c7ab2 100644
--- a/tests/unit/test_openstack.py
+++ b/tests/unit/test_openstack.py
@@ -14,14 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
from tests.base import AnsibleZuulTestCase
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestOpenStack(AnsibleZuulTestCase):
# A temporary class to experiment with how openstack can use
diff --git a/tests/unit/test_requirements.py b/tests/unit/test_requirements.py
index 1ea0b2e..7e578cf 100644
--- a/tests/unit/test_requirements.py
+++ b/tests/unit/test_requirements.py
@@ -14,15 +14,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import time
from tests.base import ZuulTestCase
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestRequirementsApprovalNewerThan(ZuulTestCase):
"""Requirements with a newer-than comment requirement"""
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 584a7e0..3b9d562 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -15,7 +15,6 @@
# under the License.
import json
-import logging
import os
import re
import shutil
@@ -36,10 +35,6 @@
repack_repo,
)
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestScheduler(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
@@ -1041,41 +1036,6 @@
self.assertIn('project-post', job_names)
@skip("Disabled for early v3 development")
- def test_build_configuration_branch(self):
- "Test that the right commits are on alternate branches"
-
- self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange('org/project', 'mp', 'A')
- B = self.fake_gerrit.addFakeChange('org/project', 'mp', 'B')
- C = self.fake_gerrit.addFakeChange('org/project', 'mp', 'C')
- A.addApproval('code-review', 2)
- B.addApproval('code-review', 2)
- C.addApproval('code-review', 2)
- self.fake_gerrit.addEvent(A.addApproval('approved', 1))
- self.fake_gerrit.addEvent(B.addApproval('approved', 1))
- self.fake_gerrit.addEvent(C.addApproval('approved', 1))
- self.waitUntilSettled()
-
- self.gearman_server.release('.*-merge')
- self.waitUntilSettled()
- self.gearman_server.release('.*-merge')
- self.waitUntilSettled()
- self.gearman_server.release('.*-merge')
- self.waitUntilSettled()
- queue = self.gearman_server.getQueue()
- ref = self.getParameter(queue[-1], 'ZUUL_REF')
- self.gearman_server.hold_jobs_in_queue = False
- self.gearman_server.release()
- self.waitUntilSettled()
-
- path = os.path.join(self.git_root, "org/project")
- repo = git.Repo(path)
- repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
- repo_messages.reverse()
- correct_messages = ['initial commit', 'mp commit', 'A-1', 'B-1', 'C-1']
- self.assertEqual(repo_messages, correct_messages)
-
- @skip("Disabled for early v3 development")
def test_build_configuration_branch_interaction(self):
"Test that switching between branches works"
self.test_build_configuration()
@@ -1086,89 +1046,6 @@
repo.heads.master.commit = repo.commit('init')
self.test_build_configuration()
- @skip("Disabled for early v3 development")
- def test_build_configuration_multi_branch(self):
- "Test that dependent changes on multiple branches are merged"
-
- self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project', 'mp', 'B')
- C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
- A.addApproval('code-review', 2)
- B.addApproval('code-review', 2)
- C.addApproval('code-review', 2)
- self.fake_gerrit.addEvent(A.addApproval('approved', 1))
- self.fake_gerrit.addEvent(B.addApproval('approved', 1))
- self.fake_gerrit.addEvent(C.addApproval('approved', 1))
- self.waitUntilSettled()
- queue = self.gearman_server.getQueue()
- job_A = None
- for job in queue:
- if 'project-merge' in job.name:
- job_A = job
- ref_A = self.getParameter(job_A, 'ZUUL_REF')
- commit_A = self.getParameter(job_A, 'ZUUL_COMMIT')
- self.log.debug("Got Zuul ref for change A: %s" % ref_A)
- self.log.debug("Got Zuul commit for change A: %s" % commit_A)
-
- self.gearman_server.release('.*-merge')
- self.waitUntilSettled()
- queue = self.gearman_server.getQueue()
- job_B = None
- for job in queue:
- if 'project-merge' in job.name:
- job_B = job
- ref_B = self.getParameter(job_B, 'ZUUL_REF')
- commit_B = self.getParameter(job_B, 'ZUUL_COMMIT')
- self.log.debug("Got Zuul ref for change B: %s" % ref_B)
- self.log.debug("Got Zuul commit for change B: %s" % commit_B)
-
- self.gearman_server.release('.*-merge')
- self.waitUntilSettled()
- queue = self.gearman_server.getQueue()
- for job in queue:
- if 'project-merge' in job.name:
- job_C = job
- ref_C = self.getParameter(job_C, 'ZUUL_REF')
- commit_C = self.getParameter(job_C, 'ZUUL_COMMIT')
- self.log.debug("Got Zuul ref for change C: %s" % ref_C)
- self.log.debug("Got Zuul commit for change C: %s" % commit_C)
- self.gearman_server.hold_jobs_in_queue = False
- self.gearman_server.release()
- self.waitUntilSettled()
-
- path = os.path.join(self.git_root, "org/project")
- repo = git.Repo(path)
-
- repo_messages = [c.message.strip()
- for c in repo.iter_commits(ref_C)]
- repo_shas = [c.hexsha for c in repo.iter_commits(ref_C)]
- repo_messages.reverse()
- correct_messages = ['initial commit', 'A-1', 'C-1']
- # Ensure the right commits are in the history for this ref
- self.assertEqual(repo_messages, correct_messages)
- # Ensure ZUUL_REF -> ZUUL_COMMIT
- self.assertEqual(repo_shas[0], commit_C)
-
- repo_messages = [c.message.strip()
- for c in repo.iter_commits(ref_B)]
- repo_shas = [c.hexsha for c in repo.iter_commits(ref_B)]
- repo_messages.reverse()
- correct_messages = ['initial commit', 'mp commit', 'B-1']
- self.assertEqual(repo_messages, correct_messages)
- self.assertEqual(repo_shas[0], commit_B)
-
- repo_messages = [c.message.strip()
- for c in repo.iter_commits(ref_A)]
- repo_shas = [c.hexsha for c in repo.iter_commits(ref_A)]
- repo_messages.reverse()
- correct_messages = ['initial commit', 'A-1']
- self.assertEqual(repo_messages, correct_messages)
- self.assertEqual(repo_shas[0], commit_A)
-
- self.assertNotEqual(ref_A, ref_B, ref_C)
- self.assertNotEqual(commit_A, commit_B, commit_C)
-
def test_dependent_changes_dequeue(self):
"Test that dependent patches are not needlessly tested"
@@ -2043,11 +1920,11 @@
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
- self.builds[0].run_error = True
+ self.builds[0].requeue = True
self.launch_server.hold_jobs_in_build = False
self.launch_server.release()
self.waitUntilSettled()
- self.assertEqual(self.countJobResults(self.history, 'RUN_ERROR'), 1)
+ self.assertEqual(self.countJobResults(self.history, None), 1)
self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
def test_statsd(self):
@@ -2114,11 +1991,10 @@
self.assertIn('Build succeeded', A.messages[0])
self.assertIn('Build succeeded', B.messages[0])
- @skip("Disabled for early v3 development")
def test_file_jobs(self):
"Test that file jobs run only when appropriate"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
- A.addPatchset(['pip-requires'])
+ A.addPatchset({'pip-requires': 'foo'})
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
@@ -2305,11 +2181,9 @@
self.sched.reconfigure(self.config)
self.assertEqual(len(self.sched.layout.pipelines['gate'].queues), 1)
- @skip("Disabled for early v3 development")
def test_mutex(self):
"Test job mutexes"
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-mutex.yaml')
+ self.updateConfigLayout('layout-mutex')
self.sched.reconfigure(self.config)
self.launch_server.hold_jobs_in_build = True
@@ -2394,6 +2268,7 @@
self.waitUntilSettled()
self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
self.launch_server.hold_jobs_in_build = False
self.launch_server.release()
@@ -4699,8 +4574,8 @@
body[3])
-class TestSchedulerMergeModes(ZuulTestCase):
- tenant_config_file = 'config/merge-modes/main.yaml'
+class TestSchedulerMerges(ZuulTestCase):
+ tenant_config_file = 'config/merges/main.yaml'
def _test_project_merge_mode(self, mode):
self.launch_server.keep_jobdir = False
@@ -4760,3 +4635,137 @@
'C-1']
result = self._test_project_merge_mode('cherry-pick')
self.assertEqual(result, expected_messages)
+
+ def test_merge_branch(self):
+ "Test that the right commits are on alternate branches"
+ self.create_branch('org/project-merge-branches', 'mp')
+
+ self.launch_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange(
+ 'org/project-merge-branches', 'mp', 'A')
+ B = self.fake_gerrit.addFakeChange(
+ 'org/project-merge-branches', 'mp', 'B')
+ C = self.fake_gerrit.addFakeChange(
+ 'org/project-merge-branches', 'mp', 'C')
+ A.addApproval('code-review', 2)
+ B.addApproval('code-review', 2)
+ C.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(C.addApproval('approved', 1))
+ self.waitUntilSettled()
+
+ self.launch_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.launch_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.launch_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ build = self.builds[-1]
+ self.assertEqual(self.getParameter(build, 'ZUUL_BRANCH'), 'mp')
+ ref = self.getParameter(build, 'ZUUL_REF')
+ path = os.path.join(
+ build.jobdir.git_root, 'org/project-merge-branches')
+ repo = git.Repo(path)
+
+ repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
+ repo_messages.reverse()
+ correct_messages = [
+ 'initial commit',
+ 'add content from fixture',
+ 'mp commit',
+ 'A-1', 'B-1', 'C-1']
+ self.assertEqual(repo_messages, correct_messages)
+
+ self.launch_server.hold_jobs_in_build = False
+ self.launch_server.release()
+ self.waitUntilSettled()
+
+ def test_merge_multi_branch(self):
+ "Test that dependent changes on multiple branches are merged"
+ self.create_branch('org/project-merge-branches', 'mp')
+
+ self.launch_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange(
+ 'org/project-merge-branches', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange(
+ 'org/project-merge-branches', 'mp', 'B')
+ C = self.fake_gerrit.addFakeChange(
+ 'org/project-merge-branches', 'master', 'C')
+ A.addApproval('code-review', 2)
+ B.addApproval('code-review', 2)
+ C.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(C.addApproval('approved', 1))
+ self.waitUntilSettled()
+
+ job_A = None
+ for job in self.builds:
+ if 'project-merge' in job.name:
+ job_A = job
+ ref_A = self.getParameter(job_A, 'ZUUL_REF')
+ commit_A = self.getParameter(job_A, 'ZUUL_COMMIT')
+ self.log.debug("Got Zuul ref for change A: %s" % ref_A)
+ self.log.debug("Got Zuul commit for change A: %s" % commit_A)
+
+ path = os.path.join(
+ job_A.jobdir.git_root, "org/project-merge-branches")
+ repo = git.Repo(path)
+ repo_messages = [c.message.strip()
+ for c in repo.iter_commits(ref_A)]
+ repo_messages.reverse()
+ correct_messages = [
+ 'initial commit', 'add content from fixture', 'A-1']
+ self.assertEqual(repo_messages, correct_messages)
+
+ self.launch_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ job_B = None
+ for job in self.builds:
+ if 'project-merge' in job.name:
+ job_B = job
+ ref_B = self.getParameter(job_B, 'ZUUL_REF')
+ commit_B = self.getParameter(job_B, 'ZUUL_COMMIT')
+ self.log.debug("Got Zuul ref for change B: %s" % ref_B)
+ self.log.debug("Got Zuul commit for change B: %s" % commit_B)
+
+ path = os.path.join(
+ job_B.jobdir.git_root, "org/project-merge-branches")
+ repo = git.Repo(path)
+ repo_messages = [c.message.strip()
+ for c in repo.iter_commits(ref_B)]
+ repo_messages.reverse()
+ correct_messages = [
+ 'initial commit', 'add content from fixture', 'mp commit', 'B-1']
+ self.assertEqual(repo_messages, correct_messages)
+
+ self.launch_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ job_C = None
+ for job in self.builds:
+ if 'project-merge' in job.name:
+ job_C = job
+ ref_C = self.getParameter(job_C, 'ZUUL_REF')
+ commit_C = self.getParameter(job_C, 'ZUUL_COMMIT')
+ self.log.debug("Got Zuul ref for change C: %s" % ref_C)
+ self.log.debug("Got Zuul commit for change C: %s" % commit_C)
+ path = os.path.join(
+ job_C.jobdir.git_root, "org/project-merge-branches")
+ repo = git.Repo(path)
+ repo_messages = [c.message.strip()
+ for c in repo.iter_commits(ref_C)]
+
+ repo_messages.reverse()
+ correct_messages = [
+ 'initial commit', 'add content from fixture',
+ 'A-1', 'C-1']
+ # Ensure the right commits are in the history for this ref
+ self.assertEqual(repo_messages, correct_messages)
+
+ self.launch_server.hold_jobs_in_build = False
+ self.launch_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 8853302..0ba5ff8 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -14,15 +14,11 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
+import os
import textwrap
from tests.base import AnsibleZuulTestCase
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestMultipleTenants(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
@@ -98,8 +94,16 @@
- project-test2
""")
+ in_repo_playbook = textwrap.dedent(
+ """
+ - hosts: all
+ tasks: []
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf,
+ 'playbooks/project-test2.yaml': in_repo_playbook}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
- files={'.zuul.yaml': in_repo_conf})
+ files=file_dict)
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
@@ -110,3 +114,24 @@
"A should report start and success")
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")
+
+
+class TestAnsible(AnsibleZuulTestCase):
+ # A temporary class to hold new tests while others are disabled
+
+ tenant_config_file = 'config/ansible/main.yaml'
+
+ def test_playbook(self):
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ build = self.getJobFromHistory('python27')
+ self.assertEqual(build.result, 'SUCCESS')
+ flag_path = os.path.join(self.test_root, build.uuid + '.flag')
+ self.assertTrue(os.path.exists(flag_path))
+ pre_flag_path = os.path.join(self.test_root, build.uuid +
+ '.pre.flag')
+ self.assertTrue(os.path.exists(pre_flag_path))
+ post_flag_path = os.path.join(self.test_root, build.uuid +
+ '.post.flag')
+ self.assertTrue(os.path.exists(post_flag_path))
diff --git a/tests/unit/test_zuultrigger.py b/tests/unit/test_zuultrigger.py
index b36e5a4..5d9c6e0 100644
--- a/tests/unit/test_zuultrigger.py
+++ b/tests/unit/test_zuultrigger.py
@@ -14,14 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
from tests.base import ZuulTestCase
-logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(name)-32s '
- '%(levelname)-8s %(message)s')
-
class TestZuulTriggerParentChangeEnqueued(ZuulTestCase):
tenant_config_file = 'config/zuultrigger/parent-change-enqueued/main.yaml'
diff --git a/tox.ini b/tox.ini
index b7cbf27..b7d89d1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,9 +8,8 @@
setenv = STATSD_HOST=127.0.0.1
STATSD_PORT=8125
VIRTUAL_ENV={envdir}
- OS_TEST_TIMEOUT=30
- OS_LOG_DEFAULTS={env:OS_LOG_DEFAULTS:gear=WARNING}
-passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE
+ OS_TEST_TIMEOUT=60
+passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS
usedevelop = True
install_command = pip install {opts} {packages}
deps = -r{toxinidir}/requirements.txt
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 7b35a86..885e6b3 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -101,8 +101,9 @@
'nodes': vs.Any([node], str),
'timeout': int,
'attempts': int,
- '_source_project': model.Project,
- '_source_branch': vs.Any(str, None),
+ 'pre-run': to_list(str),
+ 'post-run': to_list(str),
+ '_source_context': model.SourceContext,
}
return vs.Schema(job)
@@ -110,16 +111,15 @@
@staticmethod
def fromYaml(layout, conf):
JobParser.getSchema()(conf)
+
job = model.Job(conf['name'])
if 'auth' in conf:
job.auth = conf.get('auth')
if 'parent' in conf:
parent = layout.getJob(conf['parent'])
- job.inheritFrom(parent)
+ job.inheritFrom(parent, 'parent while parsing')
job.timeout = conf.get('timeout', job.timeout)
job.workspace = conf.get('workspace', job.workspace)
- job.pre_run = as_list(conf.get('pre-run', job.pre_run))
- job.post_run = as_list(conf.get('post-run', job.post_run))
job.voting = conf.get('voting', True)
job.hold_following_changes = conf.get('hold-following-changes', False)
job.mutex = conf.get('mutex', None)
@@ -143,12 +143,30 @@
# accumulate onto any previously applied tags from
# metajobs.
job.tags = job.tags.union(set(tags))
- # The source attributes may not be overridden -- they are
- # always supplied by the config loader. They correspond to
- # the Project instance of the repo where it originated, and
- # the branch name.
- job.source_project = conf.get('_source_project')
- job.source_branch = conf.get('_source_branch')
+ # The source attribute and playbook info may not be
+ # overridden -- they are always supplied by the config loader.
+ # They correspond to the Project instance of the repo where it
+ # originated, and the branch name.
+ job.source_context = conf.get('_source_context')
+ pre_run_name = conf.get('pre-run')
+ # Append the pre-run command
+ if pre_run_name:
+ pre_run_name = os.path.join('playbooks', pre_run_name)
+ pre_run = model.PlaybookContext(job.source_context,
+ pre_run_name)
+ job.pre_run.append(pre_run)
+ # Prepend the post-run command
+ post_run_name = conf.get('post-run')
+ if post_run_name:
+ post_run_name = os.path.join('playbooks', post_run_name)
+ post_run = model.PlaybookContext(job.source_context,
+ post_run_name)
+ job.post_run.insert(0, post_run)
+ # Set the run command
+ run_name = job.name
+ run_name = os.path.join('playbooks', run_name)
+ run = model.PlaybookContext(job.source_context, run_name)
+ job.run = run
job.failure_message = conf.get('failure-message', job.failure_message)
job.success_message = conf.get('success-message', job.success_message)
job.failure_url = conf.get('failure-url', job.failure_url)
@@ -156,8 +174,8 @@
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
- if job.source_branch:
- branches = [job.source_branch]
+ if (not job.source_context.secure):
+ branches = [job.source_context.branch]
elif 'branches' in conf:
branches = as_list(conf['branches'])
else:
@@ -190,7 +208,9 @@
vs.Required('name'): str,
'merge-mode': vs.Any(
'merge', 'merge-resolve',
- 'cherry-pick')}
+ 'cherry-pick'),
+ '_source_context': model.SourceContext,
+ }
for p in layout.pipelines.values():
project_template[p.name] = {'queue': str,
@@ -201,6 +221,7 @@
def fromYaml(layout, conf):
ProjectTemplateParser.getSchema(layout)(conf)
project_template = model.ProjectConfig(conf['name'])
+ source_context = conf['_source_context']
for pipeline in layout.pipelines.values():
conf_pipeline = conf.get(pipeline.name)
if not conf_pipeline:
@@ -209,11 +230,12 @@
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
- layout, conf_pipeline.get('jobs', []))
+ layout, conf_pipeline.get('jobs', []),
+ source_context)
return project_template
@staticmethod
- def _parseJobTree(layout, conf, tree=None):
+ def _parseJobTree(layout, conf, source_context, tree=None):
if not tree:
tree = model.JobTree(None)
for conf_job in conf:
@@ -227,6 +249,7 @@
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
+ attrs['_source_context'] = source_context
subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
else:
# Not overriding, so get existing job
@@ -234,7 +257,9 @@
if jobs:
# This is the root of a sub tree
- ProjectTemplateParser._parseJobTree(layout, jobs, subtree)
+ ProjectTemplateParser._parseJobTree(layout, jobs,
+ source_context,
+ subtree)
else:
raise Exception("Job must be a string or dictionary")
return tree
@@ -245,10 +270,14 @@
@staticmethod
def getSchema(layout):
- project = {vs.Required('name'): str,
- 'templates': [str],
- 'merge-mode': vs.Any('merge', 'merge-resolve',
- 'cherry-pick')}
+ project = {
+ vs.Required('name'): str,
+ 'templates': [str],
+ 'merge-mode': vs.Any('merge', 'merge-resolve',
+ 'cherry-pick'),
+ '_source_context': model.SourceContext,
+ }
+
for p in layout.pipelines.values():
project[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
@@ -284,7 +313,8 @@
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_tree.inheritFrom(
- template_pipeline.job_tree)
+ template_pipeline.job_tree,
+ 'job tree while parsing')
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if queue_name:
@@ -545,8 +575,7 @@
url = source.getGitUrl(project)
job = merger.getFiles(project.name, url, 'master',
files=['zuul.yaml', '.zuul.yaml'])
- job.project = project
- job.config_repo = True
+ job.source_context = model.SourceContext(project, 'master', True)
jobs.append(job)
for (source, project) in project_repos:
@@ -560,9 +589,8 @@
for branch in source.getProjectBranches(project):
job = merger.getFiles(project.name, url, branch,
files=['.zuul.yaml'])
- job.project = project
- job.branch = branch
- job.config_repo = False
+ job.source_context = model.SourceContext(project,
+ branch, False)
jobs.append(job)
for job in jobs:
@@ -576,32 +604,31 @@
if job.files.get(fn):
TenantParser.log.info(
"Loading configuration from %s/%s" %
- (job.project, fn))
- if job.config_repo:
+ (job.source_context, fn))
+ if job.source_context.secure:
incdata = TenantParser._parseConfigRepoLayout(
- job.files[fn], job.project)
+ job.files[fn], job.source_context)
config_repos_config.extend(incdata)
else:
incdata = TenantParser._parseProjectRepoLayout(
- job.files[fn], job.project, job.branch)
+ job.files[fn], job.source_context)
project_repos_config.extend(incdata)
- job.project.unparsed_config = incdata
+ job.source_context.project.unparsed_config = incdata
return config_repos_config, project_repos_config
@staticmethod
- def _parseConfigRepoLayout(data, project):
+ def _parseConfigRepoLayout(data, source_context):
# This is the top-level configuration for a tenant.
config = model.UnparsedTenantConfig()
- config.extend(yaml.load(data), project)
-
+ config.extend(yaml.load(data), source_context)
return config
@staticmethod
- def _parseProjectRepoLayout(data, project, branch):
+ def _parseProjectRepoLayout(data, source_context):
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
config = model.UnparsedTenantConfig()
- config.extend(yaml.load(data), project, branch)
+ config.extend(yaml.load(data), source_context)
return config
@@ -672,8 +699,10 @@
data = project.unparsed_config
if not data:
continue
+ source_context = model.SourceContext(project,
+ branch, False)
incdata = TenantParser._parseProjectRepoLayout(
- data, project, branch)
+ data, source_context)
config.extend(incdata)
layout = model.Layout()
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index ac644eb..627c716 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -255,7 +255,7 @@
def getProject(self, name):
if name not in self.projects:
- self.projects[name] = Project(name)
+ self.projects[name] = Project(name, self.connection_name)
return self.projects[name]
def maintainCache(self, relevant):
diff --git a/zuul/driver/timer/__init__.py b/zuul/driver/timer/__init__.py
index 4db779b..b8979c4 100644
--- a/zuul/driver/timer/__init__.py
+++ b/zuul/driver/timer/__init__.py
@@ -76,12 +76,12 @@
jobs.append(job)
def _onTrigger(self, tenant, pipeline_name, timespec):
- for project in tenant.layout.projects.values():
+ for project_name in tenant.layout.project_configs.keys():
event = TriggerEvent()
event.type = 'timer'
event.timespec = timespec
event.forced_pipeline = pipeline_name
- event.project_name = project.name
+ event.project_name = project_name
self.log.debug("Adding event %s" % event)
self.sched.addEvent(event)
diff --git a/zuul/launcher/client.py b/zuul/launcher/client.py
index 9e895ef..458aeaf 100644
--- a/zuul/launcher/client.py
+++ b/zuul/launcher/client.py
@@ -98,7 +98,7 @@
class ZuulGearmanClient(gear.Client):
def __init__(self, zuul_gearman):
- super(ZuulGearmanClient, self).__init__()
+ super(ZuulGearmanClient, self).__init__('Zuul Launch Client')
self.__zuul_gearman = zuul_gearman
def handleWorkComplete(self, packet):
@@ -300,6 +300,10 @@
[x.change for x in dependent_items]))
dependent_items = dependent_items[:]
dependent_items.reverse()
+ # TODOv3(jeblair): This ansible vars data structure will
+ # replace the environment variables below.
+ zuul_params = dict(uuid=uuid)
+ # Legacy environment variables
params = dict(ZUUL_UUID=uuid,
ZUUL_PROJECT=item.change.project.name)
params['ZUUL_PIPELINE'] = pipeline.name
@@ -368,10 +372,17 @@
params['job'] = job.name
params['items'] = merger_items
params['projects'] = []
+
+ if job.name != 'noop':
+ params['playbook'] = job.run.toDict()
+ params['pre_playbooks'] = [x.toDict() for x in job.pre_run]
+ params['post_playbooks'] = [x.toDict() for x in job.post_run]
+
nodes = []
for node in item.current_build_set.getJobNodeSet(job.name).getNodes():
nodes.append(dict(name=node.name, image=node.image))
params['nodes'] = nodes
+ params['zuul'] = zuul_params
projects = set()
for item in all_items:
if item.change.project not in projects:
diff --git a/zuul/launcher/server.py b/zuul/launcher/server.py
index bfcc8a4..4e0fdd2 100644
--- a/zuul/launcher/server.py
+++ b/zuul/launcher/server.py
@@ -24,9 +24,9 @@
import threading
import time
import traceback
+import yaml
import gear
-import yaml
import zuul.merger
import zuul.ansible.library
@@ -66,27 +66,57 @@
# repos end up in git.openstack.org.
+class JobDirPlaybook(object):
+ def __init__(self, root):
+ self.root = root
+ self.secure = None
+ self.path = None
+
+
class JobDir(object):
- def __init__(self, keep=False):
+ def __init__(self, root=None, keep=False):
self.keep = keep
- self.root = tempfile.mkdtemp()
+ self.root = tempfile.mkdtemp(dir=root)
self.git_root = os.path.join(self.root, 'git')
os.makedirs(self.git_root)
self.ansible_root = os.path.join(self.root, 'ansible')
os.makedirs(self.ansible_root)
self.known_hosts = os.path.join(self.ansible_root, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory')
- self.playbook = os.path.join(self.ansible_root, 'playbook')
- self.post_playbook = os.path.join(self.ansible_root, 'post_playbook')
+ self.vars = os.path.join(self.ansible_root, 'vars.yaml')
+ self.playbook_root = os.path.join(self.ansible_root, 'playbook')
+ os.makedirs(self.playbook_root)
+ self.playbook = JobDirPlaybook(self.playbook_root)
+ self.pre_playbooks = []
+ self.post_playbooks = []
self.config = os.path.join(self.ansible_root, 'ansible.cfg')
self.ansible_log = os.path.join(self.ansible_root, 'ansible_log.txt')
+ def addPrePlaybook(self):
+ count = len(self.pre_playbooks)
+ root = os.path.join(self.ansible_root, 'pre_playbook_%i' % (count,))
+ os.makedirs(root)
+ playbook = JobDirPlaybook(root)
+ self.pre_playbooks.append(playbook)
+ return playbook
+
+ def addPostPlaybook(self):
+ count = len(self.post_playbooks)
+ root = os.path.join(self.ansible_root, 'post_playbook_%i' % (count,))
+ os.makedirs(root)
+ playbook = JobDirPlaybook(root)
+ self.post_playbooks.append(playbook)
+ return playbook
+
+ def cleanup(self):
+ if not self.keep:
+ shutil.rmtree(self.root)
+
def __enter__(self):
return self
def __exit__(self, etype, value, tb):
- if not self.keep:
- shutil.rmtree(self.root)
+ self.cleanup()
class UpdateTask(object):
@@ -149,9 +179,11 @@
class LaunchServer(object):
log = logging.getLogger("zuul.LaunchServer")
- def __init__(self, config, connections={}, keep_jobdir=False):
+ def __init__(self, config, connections={}, jobdir_root=None,
+ keep_jobdir=False):
self.config = config
self.keep_jobdir = keep_jobdir
+ self.jobdir_root = jobdir_root
# TODOv3(mordred): make the launcher name more unique --
# perhaps hostname+pid.
self.hostname = socket.gethostname()
@@ -179,10 +211,6 @@
self.merge_name = self.config.get('merger', 'git_user_name')
else:
self.merge_name = None
- if self.config.has_option('launcher', 'private_key_file'):
- self.private_key_file = config.get('launcher', 'private_key_file')
- else:
- self.private_key_file = '~/.ssh/id_rsa'
self.connections = connections
self.merger = self._getMerger(self.merge_root)
@@ -205,6 +233,8 @@
for fn in os.listdir(library_path):
shutil.copy(os.path.join(library_path, fn), self.library_dir)
+ self.job_workers = {}
+
def _getMerger(self, root):
return zuul.merger.merger.Merger(root, self.connections,
self.merge_email, self.merge_name)
@@ -334,184 +364,28 @@
self.log.exception("Exception while getting job")
def launchJob(self, job):
- thread = threading.Thread(target=self._launch, args=(job,))
- thread.start()
+ self.job_workers[job.unique] = AnsibleJob(self, job)
+ self.job_workers[job.unique].run()
- def _launch(self, job):
- self.log.debug("Job %s: beginning" % (job.unique,))
- with JobDir() as jobdir:
- self.log.debug("Job %s: job root at %s" %
- (job.unique, jobdir.root))
- args = json.loads(job.arguments)
- tasks = []
- for project in args['projects']:
- self.log.debug("Job %s: updating project %s" %
- (job.unique, project['name']))
- tasks.append(self.update(project['name'], project['url']))
- for task in tasks:
- task.wait()
- self.log.debug("Job %s: git updates complete" % (job.unique,))
- merger = self._getMerger(jobdir.git_root)
- merge_items = [i for i in args['items'] if i.get('refspec')]
- if merge_items:
- commit = merger.mergeChanges(merge_items) # noqa
- else:
- commit = args['items'][-1]['newrev'] # noqa
-
- # TODOv3: Ansible the ansible thing here.
- self.prepareAnsibleFiles(jobdir, args)
-
- data = {
- 'manager': self.hostname,
- 'url': 'https://server/job/{}/0/'.format(args['job']),
- 'worker_name': 'My Worker',
- }
-
- # TODOv3:
- # 'name': self.name,
- # 'manager': self.launch_server.hostname,
- # 'worker_name': 'My Worker',
- # 'worker_hostname': 'localhost',
- # 'worker_ips': ['127.0.0.1', '192.168.1.1'],
- # 'worker_fqdn': 'zuul.example.org',
- # 'worker_program': 'FakeBuilder',
- # 'worker_version': 'v1.1',
- # 'worker_extra': {'something': 'else'}
-
- job.sendWorkData(json.dumps(data))
- job.sendWorkStatus(0, 100)
-
- result = self.runAnsible(jobdir, job)
- if result is None:
- job.sendWorkFail()
- return
- result = dict(result=result)
- job.sendWorkComplete(json.dumps(result))
+ def finishJob(self, unique):
+ del(self.job_workers[unique])
def stopJob(self, job):
- # TODOv3: implement.
- job.sendWorkComplete()
-
- def getHostList(self, args):
- # TODOv3: the localhost addition is temporary so we have
- # something to exercise ansible.
- hosts = [('localhost', dict(ansible_connection='local'))]
- for node in args['nodes']:
- # TODOv3: the connection should almost certainly not be
- # local.
- hosts.append((node['name'], dict(ansible_connection='local')))
- return hosts
-
- def prepareAnsibleFiles(self, jobdir, args):
- with open(jobdir.inventory, 'w') as inventory:
- for host_name, host_vars in self.getHostList(args):
- inventory.write(host_name)
- inventory.write(' ')
- for k, v in host_vars.items():
- inventory.write('%s=%s' % (k, v))
- inventory.write('\n')
- with open(jobdir.playbook, 'w') as playbook:
- play = dict(hosts='localhost',
- tasks=[dict(name='test',
- shell='echo Hello world')])
- playbook.write(yaml.dump([play]))
- with open(jobdir.config, 'w') as config:
- config.write('[defaults]\n')
- config.write('hostfile = %s\n' % jobdir.inventory)
- config.write('local_tmp = %s/.ansible/local_tmp\n' % jobdir.root)
- config.write('remote_tmp = %s/.ansible/remote_tmp\n' % jobdir.root)
- config.write('private_key_file = %s\n' % self.private_key_file)
- config.write('retry_files_enabled = False\n')
- config.write('log_path = %s\n' % jobdir.ansible_log)
- config.write('gathering = explicit\n')
- config.write('library = %s\n' % self.library_dir)
- # bump the timeout because busy nodes may take more than
- # 10s to respond
- config.write('timeout = 30\n')
-
- config.write('[ssh_connection]\n')
- # NB: when setting pipelining = True, keep_remote_files
- # must be False (the default). Otherwise it apparently
- # will override the pipelining option and effectively
- # disable it. Pipelining has a side effect of running the
- # command without a tty (ie, without the -tt argument to
- # ssh). We require this behavior so that if a job runs a
- # command which expects interactive input on a tty (such
- # as sudo) it does not hang.
- config.write('pipelining = True\n')
- ssh_args = "-o ControlMaster=auto -o ControlPersist=60s " \
- "-o UserKnownHostsFile=%s" % jobdir.known_hosts
- config.write('ssh_args = %s\n' % ssh_args)
-
- def _ansibleTimeout(self, proc, msg):
- self.log.warning(msg)
- self.abortRunningProc(proc)
-
- def abortRunningProc(self, proc):
- aborted = False
- self.log.debug("Abort: sending kill signal to job "
- "process group")
try:
- pgid = os.getpgid(proc.pid)
- os.killpg(pgid, signal.SIGKILL)
- aborted = True
- except Exception:
- self.log.exception("Exception while killing "
- "ansible process:")
- return aborted
-
- def runAnsible(self, jobdir, job):
- # Job is included here for the benefit of the test framework.
- env_copy = os.environ.copy()
- env_copy['LOGNAME'] = 'zuul'
-
- if False: # TODOv3: self.options['verbose']:
- verbose = '-vvv'
- else:
- verbose = '-v'
-
- cmd = ['ansible-playbook', jobdir.playbook, verbose]
- self.log.debug("Ansible command: %s" % (cmd,))
- # TODOv3: verbose
- proc = subprocess.Popen(
- cmd,
- cwd=jobdir.ansible_root,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- preexec_fn=os.setsid,
- env=env_copy,
- )
-
- ret = None
- # TODOv3: get this from the job
- timeout = 60
- watchdog = Watchdog(timeout + ANSIBLE_WATCHDOG_GRACE,
- self._ansibleTimeout,
- (proc,
- "Ansible timeout exceeded"))
- watchdog.start()
- try:
- for line in iter(proc.stdout.readline, b''):
- line = line[:1024].rstrip()
- self.log.debug("Ansible output: %s" % (line,))
- ret = proc.wait()
+ args = json.loads(job.arguments)
+ self.log.debug("Stop job with arguments: %s" % (args,))
+ unique = args['uuid']
+ job_worker = self.job_workers.get(unique)
+ if not job_worker:
+ self.log.debug("Unable to find worker for job %s" % (unique,))
+ return
+ try:
+ job_worker.stop()
+ except Exception:
+ self.log.exception("Exception sending stop command "
+ "to worker:")
finally:
- watchdog.stop()
- self.log.debug("Ansible exit code: %s" % (ret,))
-
- if watchdog.timed_out:
- return 'TIMED_OUT'
- if ret == 3:
- # AnsibleHostUnreachable: We had a network issue connecting to
- # our zuul-worker.
- return None
- elif ret == -9:
- # Received abort request.
- return None
-
- if ret == 0:
- return 'SUCCESS'
- return 'FAILURE'
+ job.sendWorkComplete()
def cat(self, job):
args = json.loads(job.arguments)
@@ -534,3 +408,338 @@
else:
result['commit'] = ret
job.sendWorkComplete(json.dumps(result))
+
+
+class AnsibleJob(object):
+ log = logging.getLogger("zuul.AnsibleJob")
+
+ RESULT_NORMAL = 1
+ RESULT_TIMED_OUT = 2
+ RESULT_UNREACHABLE = 3
+ RESULT_ABORTED = 4
+
+ def __init__(self, launcher_server, job):
+ self.launcher_server = launcher_server
+ self.job = job
+ self.jobdir = None
+ self.proc = None
+ self.proc_lock = threading.Lock()
+ self.running = False
+ self.aborted = False
+
+ if self.launcher_server.config.has_option(
+ 'launcher', 'private_key_file'):
+ self.private_key_file = self.launcher_server.config.get(
+ 'launcher', 'private_key_file')
+ else:
+ self.private_key_file = '~/.ssh/id_rsa'
+
+ def run(self):
+ self.running = True
+ self.thread = threading.Thread(target=self.launch)
+ self.thread.start()
+
+ def stop(self):
+ self.aborted = True
+ self.abortRunningProc()
+ self.thread.join()
+
+ def launch(self):
+ try:
+ self.jobdir = JobDir(root=self.launcher_server.jobdir_root)
+ self._launch()
+ except Exception:
+ self.log.exception("Exception while launching job")
+ self.job.sendWorkException(traceback.format_exc())
+ finally:
+ self.running = False
+ try:
+ self.jobdir.cleanup()
+ except Exception:
+ self.log.exception("Error cleaning up jobdir:")
+ try:
+ self.launcher_server.finishJob(self.job.unique)
+ except Exception:
+ self.log.exception("Error finalizing job thread:")
+
+ def _launch(self):
+ self.log.debug("Job %s: beginning" % (self.job.unique,))
+ self.log.debug("Job %s: args: %s" % (self.job.unique,
+ self.job.arguments,))
+ self.log.debug("Job %s: job root at %s" %
+ (self.job.unique, self.jobdir.root))
+ args = json.loads(self.job.arguments)
+ tasks = []
+ for project in args['projects']:
+ self.log.debug("Job %s: updating project %s" %
+ (self.job.unique, project['name']))
+ tasks.append(self.launcher_server.update(
+ project['name'], project['url']))
+ for task in tasks:
+ task.wait()
+
+ self.log.debug("Job %s: git updates complete" % (self.job.unique,))
+ merger = self.launcher_server._getMerger(self.jobdir.git_root)
+ merge_items = [i for i in args['items'] if i.get('refspec')]
+ if merge_items:
+ commit = merger.mergeChanges(merge_items) # noqa
+ else:
+ commit = args['items'][-1]['newrev'] # noqa
+
+ # is the playbook in a repo that we have already prepared?
+ self.preparePlaybookRepos(args)
+
+ # TODOv3: Ansible the ansible thing here.
+ self.prepareAnsibleFiles(args)
+
+ data = {
+ 'manager': self.launcher_server.hostname,
+ 'url': 'https://server/job/{}/0/'.format(args['job']),
+ 'worker_name': 'My Worker',
+ }
+
+ # TODOv3:
+ # 'name': self.name,
+ # 'manager': self.launch_server.hostname,
+ # 'worker_name': 'My Worker',
+ # 'worker_hostname': 'localhost',
+ # 'worker_ips': ['127.0.0.1', '192.168.1.1'],
+ # 'worker_fqdn': 'zuul.example.org',
+ # 'worker_program': 'FakeBuilder',
+ # 'worker_version': 'v1.1',
+ # 'worker_extra': {'something': 'else'}
+
+ self.job.sendWorkData(json.dumps(data))
+ self.job.sendWorkStatus(0, 100)
+
+ result = self.runPlaybooks()
+
+ if result is None:
+ self.job.sendWorkFail()
+ return
+ result = dict(result=result)
+ self.job.sendWorkComplete(json.dumps(result))
+
+ def runPlaybooks(self):
+ result = None
+
+ for playbook in self.jobdir.pre_playbooks:
+ pre_status, pre_code = self.runAnsiblePlaybook(playbook)
+ if pre_status != self.RESULT_NORMAL or pre_code != 0:
+ # These should really never fail, so return None and have
+ # zuul try again
+ return result
+
+ job_status, job_code = self.runAnsiblePlaybook(self.jobdir.playbook)
+ if job_status == self.RESULT_TIMED_OUT:
+ return 'TIMED_OUT'
+ if job_status == self.RESULT_ABORTED:
+ return 'ABORTED'
+ if job_status != self.RESULT_NORMAL:
+ # The result of the job is indeterminate. Zuul will
+ # run it again.
+ return result
+
+ success = (job_code == 0)
+ if success:
+ result = 'SUCCESS'
+ else:
+ result = 'FAILURE'
+
+ for playbook in self.jobdir.post_playbooks:
+ post_status, post_code = self.runAnsiblePlaybook(
+ playbook, success)
+ if post_status != self.RESULT_NORMAL or post_code != 0:
+ result = 'POST_FAILURE'
+ return result
+
+ def getHostList(self, args):
+ # TODOv3: the localhost addition is temporary so we have
+ # something to exercise ansible.
+ hosts = [('localhost', dict(ansible_connection='local'))]
+ for node in args['nodes']:
+ # TODOv3: the connection should almost certainly not be
+ # local.
+ hosts.append((node['name'], dict(ansible_connection='local')))
+ return hosts
+
+ def findPlaybook(self, path):
+ for ext in ['.yaml', '.yml']:
+ fn = path + ext
+ if os.path.exists(fn):
+ return fn
+ raise Exception("Unable to find playbook %s" % path)
+
+ def preparePlaybookRepos(self, args):
+ for playbook in args['pre_playbooks']:
+ jobdir_playbook = self.jobdir.addPrePlaybook()
+ self.preparePlaybookRepo(jobdir_playbook, playbook, args)
+
+ jobdir_playbook = self.jobdir.playbook
+ self.preparePlaybookRepo(jobdir_playbook, args['playbook'], args)
+
+ for playbook in args['post_playbooks']:
+ jobdir_playbook = self.jobdir.addPostPlaybook()
+ self.preparePlaybookRepo(jobdir_playbook, playbook, args)
+
+ def preparePlaybookRepo(self, jobdir_playbook, playbook, args):
+ # Check out the playbook repo if needed and set the path to
+ # the playbook that should be run.
+ jobdir_playbook.secure = playbook['secure']
+ source = self.launcher_server.connections.getSource(
+ playbook['connection'])
+ project = source.getProject(playbook['project'])
+ # TODO(jeblair): construct the url in the merger itself
+ url = source.getGitUrl(project)
+ if not playbook['secure']:
+ # This is a project repo, so it is safe to use the already
+ # checked out version (from speculative merging) of the
+ # playbook
+ for i in args['items']:
+ if (i['connection_name'] == playbook['connection'] and
+ i['project'] == playbook['project']):
+ # We already have this repo prepared
+ path = os.path.join(self.jobdir.git_root,
+ project.name,
+ playbook['path'])
+ jobdir_playbook.path = self.findPlaybook(path)
+ return
+ # The playbook repo is either a config repo, or it isn't in
+ # the stack of changes we are testing, so check out the branch
+ # tip into a dedicated space.
+
+ merger = self.launcher_server._getMerger(jobdir_playbook.root)
+ merger.checkoutBranch(project.name, url, playbook['branch'])
+
+ path = os.path.join(jobdir_playbook.root,
+ project.name,
+ playbook['path'])
+ jobdir_playbook.path = self.findPlaybook(path)
+
+ def prepareAnsibleFiles(self, args):
+ with open(self.jobdir.inventory, 'w') as inventory:
+ for host_name, host_vars in self.getHostList(args):
+ inventory.write(host_name)
+ inventory.write(' ')
+ for k, v in host_vars.items():
+ inventory.write('%s=%s' % (k, v))
+ inventory.write('\n')
+ with open(self.jobdir.vars, 'w') as vars_yaml:
+ zuul_vars = dict(zuul=args['zuul'])
+ vars_yaml.write(
+ yaml.safe_dump(zuul_vars, default_flow_style=False))
+ with open(self.jobdir.config, 'w') as config:
+ config.write('[defaults]\n')
+ config.write('hostfile = %s\n' % self.jobdir.inventory)
+ config.write('local_tmp = %s/.ansible/local_tmp\n' %
+ self.jobdir.root)
+ config.write('remote_tmp = %s/.ansible/remote_tmp\n' %
+ self.jobdir.root)
+ config.write('private_key_file = %s\n' % self.private_key_file)
+ config.write('retry_files_enabled = False\n')
+ config.write('log_path = %s\n' % self.jobdir.ansible_log)
+ config.write('gathering = explicit\n')
+ config.write('library = %s\n'
+ % self.launcher_server.library_dir)
+ # bump the timeout because busy nodes may take more than
+ # 10s to respond
+ config.write('timeout = 30\n')
+
+ config.write('[ssh_connection]\n')
+ # NB: when setting pipelining = True, keep_remote_files
+ # must be False (the default). Otherwise it apparently
+ # will override the pipelining option and effectively
+ # disable it. Pipelining has a side effect of running the
+ # command without a tty (ie, without the -tt argument to
+ # ssh). We require this behavior so that if a job runs a
+ # command which expects interactive input on a tty (such
+ # as sudo) it does not hang.
+ config.write('pipelining = True\n')
+ ssh_args = "-o ControlMaster=auto -o ControlPersist=60s " \
+ "-o UserKnownHostsFile=%s" % self.jobdir.known_hosts
+ config.write('ssh_args = %s\n' % ssh_args)
+
+ def _ansibleTimeout(self, msg):
+ self.log.warning(msg)
+ self.abortRunningProc()
+
+ def abortRunningProc(self):
+ with self.proc_lock:
+ if not self.proc:
+ self.log.debug("Abort: no process is running")
+ return
+ self.log.debug("Abort: sending kill signal to job "
+ "process group")
+ try:
+ pgid = os.getpgid(self.proc.pid)
+ os.killpg(pgid, signal.SIGKILL)
+ except Exception:
+ self.log.exception("Exception while killing "
+ "ansible process:")
+
+ def runAnsible(self, cmd, timeout):
+ env_copy = os.environ.copy()
+ env_copy['LOGNAME'] = 'zuul'
+
+ with self.proc_lock:
+ if self.aborted:
+ return (self.RESULT_ABORTED, None)
+ self.log.debug("Ansible command: %s" % (cmd,))
+ self.proc = subprocess.Popen(
+ cmd,
+ cwd=self.jobdir.ansible_root,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ preexec_fn=os.setsid,
+ env=env_copy,
+ )
+
+ ret = None
+ watchdog = Watchdog(timeout + ANSIBLE_WATCHDOG_GRACE,
+ self._ansibleTimeout,
+ ("Ansible timeout exceeded",))
+ watchdog.start()
+ try:
+ for line in iter(self.proc.stdout.readline, b''):
+ line = line[:1024].rstrip()
+ self.log.debug("Ansible output: %s" % (line,))
+ ret = self.proc.wait()
+ finally:
+ watchdog.stop()
+ self.log.debug("Ansible exit code: %s" % (ret,))
+
+ with self.proc_lock:
+ self.proc = None
+
+ if watchdog.timed_out:
+ return (self.RESULT_TIMED_OUT, None)
+ if ret == 3:
+ # AnsibleHostUnreachable: We had a network issue connecting to
+ # our zuul-worker.
+ return (self.RESULT_UNREACHABLE, None)
+ elif ret == -9:
+ # Received abort request.
+ return (self.RESULT_ABORTED, None)
+
+ return (self.RESULT_NORMAL, ret)
+
+ def runAnsiblePlaybook(self, playbook, success=None):
+ env_copy = os.environ.copy()
+ env_copy['LOGNAME'] = 'zuul'
+
+ if False: # TODOv3: self.options['verbose']:
+ verbose = '-vvv'
+ else:
+ verbose = '-v'
+
+ cmd = ['ansible-playbook', playbook.path]
+
+ if success is not None:
+ cmd.extend(['-e', 'success=%s' % str(bool(success))])
+
+ cmd.extend(['-e@%s' % self.jobdir.vars, verbose])
+
+ # TODOv3: get this from the job
+ timeout = 60
+
+ return self.runAnsible(cmd, timeout)
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 4ae7f35..95028e5 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -67,15 +67,16 @@
def log_jobs(tree, indent=0):
istr = ' ' + ' ' * indent
if tree.job:
+ # TODOv3(jeblair): represent matchers
efilters = ''
- for b in tree.job._branches:
- efilters += str(b)
- for f in tree.job._files:
- efilters += str(f)
- if tree.job.skip_if_matcher:
- efilters += str(tree.job.skip_if_matcher)
- if efilters:
- efilters = ' ' + efilters
+ # for b in tree.job._branches:
+ # efilters += str(b)
+ # for f in tree.job._files:
+ # efilters += str(f)
+ # if tree.job.skip_if_matcher:
+ # efilters += str(tree.job.skip_if_matcher)
+ # if efilters:
+ # efilters = ' ' + efilters
tags = []
if tree.job.hold_following_changes:
tags.append('[hold]')
@@ -89,10 +90,11 @@
for x in tree.job_trees:
log_jobs(x, indent + 2)
- for p in layout.projects.values():
- tree = self.pipeline.getJobTree(p)
+ for project_name in layout.project_configs.keys():
+ project = self.pipeline.source.getProject(project_name)
+ tree = self.pipeline.getJobTree(project)
if tree:
- self.log.info(" %s" % p)
+ self.log.info(" %s" % project)
log_jobs(tree)
self.log.info(" On start:")
self.log.info(" %s" % self.pipeline.start_actions)
@@ -398,6 +400,9 @@
old_build_set.node_requests = {}
canceled_jobs = set()
for build in old_build_set.getBuilds():
+ if build.result:
+ canceled_jobs.add(build.job.name)
+ continue
was_running = False
try:
was_running = self.sched.launcher.cancel(build)
@@ -619,13 +624,6 @@
if build.retry:
build.build_set.removeJobNodeSet(build.job.name)
- # If any jobs were skipped as a result of this build, return
- # their nodes.
- for build in build.build_set.getBuilds():
- if build.result == 'SKIPPED':
- nodeset = build.build_set.getJobNodeSet(build.job.name)
- self.sched.nodepool.returnNodeSet(nodeset)
-
return True
def onMergeCompleted(self, event):
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 1e98532..990d33e 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -33,7 +33,7 @@
class MergeGearmanClient(gear.Client):
def __init__(self, merge_client):
- super(MergeGearmanClient, self).__init__()
+ super(MergeGearmanClient, self).__init__('Zuul Merge Client')
self.__merge_client = merge_client
def handleWorkComplete(self, packet):
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 692dd83..3ab7b5f 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -260,6 +260,16 @@
except Exception:
self.log.exception("Unable to update %s", project)
+ def checkoutBranch(self, project, url, branch):
+ repo = self.getRepo(project, url)
+ if repo.hasBranch(branch):
+ self.log.info("Checking out branch %s of %s" % (branch, project))
+ head = repo.getBranchHead(branch)
+ repo.checkout(head)
+ else:
+ raise Exception("Project %s does not have branch %s" %
+ (project, branch))
+
def _mergeChange(self, item, ref):
repo = self.getRepo(item['project'], item['url'])
try:
diff --git a/zuul/model.py b/zuul/model.py
index 2c3c7b3..00740cb 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -353,8 +353,9 @@
# This makes a Project instance a unique identifier for a given
# project from a given source.
- def __init__(self, name, foreign=False):
+ def __init__(self, name, connection_name, foreign=False):
self.name = name
+ self.connection_name = connection_name
# foreign projects are those referenced in dependencies
# of layout projects, this should matter
# when deciding whether to enqueue their changes
@@ -435,6 +436,15 @@
self.name = name or ''
self.nodes = OrderedDict()
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, NodeSet):
+ return False
+ return (self.name == other.name and
+ self.nodes == other.nodes)
+
def copy(self):
n = NodeSet(self.name)
for name, node in self.nodes.items():
@@ -505,39 +515,107 @@
self.state_time = data['state_time']
+class SourceContext(object):
+ """A reference to the branch of a project in configuration.
+
+ Jobs and playbooks reference this to keep track of where they
+ originate."""
+
+ def __init__(self, project, branch, secure):
+ self.project = project
+ self.branch = branch
+ self.secure = secure
+
+ def __repr__(self):
+ return '<SourceContext %s:%s secure:%s>' % (self.project,
+ self.branch,
+ self.secure)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, SourceContext):
+ return False
+ return (self.project == other.project and
+ self.branch == other.branch and
+ self.secure == other.secure)
+
+
+class PlaybookContext(object):
+
+ """A reference to a playbook in the context of a project.
+
+ Jobs refer to objects of this class for their main, pre, and post
+ playbooks so that we can keep track of which repos and security
+ contexts are needed in order to run them."""
+
+ def __init__(self, source_context, path):
+ self.source_context = source_context
+ self.path = path
+
+ def __repr__(self):
+ return '<PlaybookContext %s %s>' % (self.source_context,
+ self.path)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, PlaybookContext):
+ return False
+ return (self.source_context == other.source_context and
+ self.path == other.path)
+
+ def toDict(self):
+ # Render to a dict to use in passing json to the launcher
+ return dict(
+ connection=self.source_context.project.connection_name,
+ project=self.source_context.project.name,
+ branch=self.source_context.branch,
+ secure=self.source_context.secure,
+ path=self.path)
+
+
class Job(object):
+
"""A Job represents the defintion of actions to perform."""
- attributes = dict(
- timeout=None,
- # variables={},
- nodeset=NodeSet(),
- auth={},
- workspace=None,
- pre_run=None,
- post_run=None,
- voting=None,
- hold_following_changes=None,
- failure_message=None,
- success_message=None,
- failure_url=None,
- success_url=None,
- # Matchers. These are separate so they can be individually
- # overidden.
- branch_matcher=None,
- file_matcher=None,
- irrelevant_file_matcher=None, # skip-if
- tags=set(),
- mutex=None,
- attempts=3,
- )
-
def __init__(self, name):
+ self.attributes = dict(
+ timeout=None,
+ # variables={},
+ nodeset=NodeSet(),
+ auth={},
+ workspace=None,
+ pre_run=[],
+ post_run=[],
+ run=None,
+ voting=None,
+ hold_following_changes=None,
+ failure_message=None,
+ success_message=None,
+ failure_url=None,
+ success_url=None,
+ # Matchers. These are separate so they can be individually
+ # overidden.
+ branch_matcher=None,
+ file_matcher=None,
+ irrelevant_file_matcher=None, # skip-if
+ tags=set(),
+ mutex=None,
+ attempts=3,
+ source_context=None,
+ inheritance_path=[],
+ )
+
self.name = name
- self.project_source = None
for k, v in self.attributes.items():
setattr(self, k, v)
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
def __eq__(self, other):
# Compare the name and all inheritable attributes to determine
# whether two jobs with the same name are identically
@@ -555,20 +633,28 @@
return self.name
def __repr__(self):
- return '<Job %s branches: %s>' % (self.name, self.branch_matcher)
+ return '<Job %s branches: %s source: %s>' % (self.name,
+ self.branch_matcher,
+ self.source_context)
- def inheritFrom(self, other):
+ def inheritFrom(self, other, comment='unknown'):
"""Copy the inheritable attributes which have been set on the other
job to this job."""
if not isinstance(other, Job):
raise Exception("Job unable to inherit from %s" % (other,))
+ self.inheritance_path.extend(other.inheritance_path)
+ self.inheritance_path.append('%s %s' % (repr(other), comment))
for k, v in self.attributes.items():
- if getattr(other, k) != v and k != 'auth':
+ if (getattr(other, k) != v and k not in
+ set(['auth', 'pre_run', 'post_run', 'inheritance_path'])):
setattr(self, k, getattr(other, k))
# Inherit auth only if explicitly allowed
if other.auth and 'inherit' in other.auth and other.auth['inherit']:
setattr(self, 'auth', getattr(other, 'auth'))
+ # Pre and post run are lists; make a copy
+ self.pre_run = other.pre_run + self.pre_run
+ self.post_run = self.post_run + other.post_run
def changeMatches(self, change):
if self.branch_matcher and not self.branch_matcher.matches(change):
@@ -624,16 +710,16 @@
return ret
return None
- def inheritFrom(self, other):
+ def inheritFrom(self, other, comment='unknown'):
if other.job:
self.job = Job(other.job.name)
- self.job.inheritFrom(other.job)
+ self.job.inheritFrom(other.job, comment)
for other_tree in other.job_trees:
this_tree = self.getJobTreeForJob(other_tree.job)
if not this_tree:
this_tree = JobTree(None)
self.job_trees.append(this_tree)
- this_tree.inheritFrom(other_tree)
+ this_tree.inheritFrom(other_tree, comment)
class Build(object):
@@ -1779,7 +1865,7 @@
r.nodesets = copy.deepcopy(self.nodesets)
return r
- def extend(self, conf, source_project=None, source_branch=None):
+ def extend(self, conf, source_context=None):
if isinstance(conf, UnparsedTenantConfig):
self.pipelines.extend(conf.pipelines)
self.jobs.extend(conf.jobs)
@@ -1792,6 +1878,11 @@
raise Exception("Configuration items must be in the form of "
"a list of dictionaries (when parsing %s)" %
(conf,))
+
+ if source_context is None:
+ raise Exception("A source context must be provided "
+ "(when parsing %s)" % (conf,))
+
for item in conf:
if not isinstance(item, dict):
raise Exception("Configuration items must be in the form of "
@@ -1802,13 +1893,11 @@
"a single key (when parsing %s)" %
(conf,))
key, value = item.items()[0]
+ if key in ['project', 'project-template', 'job']:
+ value['_source_context'] = source_context
if key == 'project':
self.projects.append(value)
elif key == 'job':
- if source_project is not None:
- value['_source_project'] = source_project
- if source_branch is not None:
- value['_source_branch'] = source_branch
self.jobs.append(value)
elif key == 'project-template':
self.project_templates.append(value)
@@ -1827,7 +1916,6 @@
def __init__(self):
self.tenant = None
- self.projects = {}
self.project_configs = {}
self.project_templates = {}
self.pipelines = OrderedDict()
@@ -1851,13 +1939,16 @@
def addJob(self, job):
# We can have multiple variants of a job all with the same
# name, but these variants must all be defined in the same repo.
- prior_jobs = [j for j in self.getJobs(job.name)
- if j.source_project != job.source_project]
+ prior_jobs = [j for j in self.getJobs(job.name) if
+ j.source_context.project !=
+ job.source_context.project]
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
- "job %s in %s" % (job, job.source_project,
- prior_jobs[0],
- prior_jobs[0].source_project))
+ "job %s in %s" % (
+ job,
+ job.source_context.project,
+ prior_jobs[0],
+ prior_jobs[0].source_context.project))
if job.name in self.jobs:
self.jobs[job.name].append(job)
@@ -1899,7 +1990,8 @@
for variant in self.getJobs(job.name):
if variant.changeMatches(change):
if variant not in inherited:
- frozen_job.inheritFrom(variant)
+ frozen_job.inheritFrom(variant,
+ 'variant while freezing')
inherited.add(variant)
if not inherited:
# A change must match at least one defined job variant
@@ -1910,7 +2002,7 @@
# Only update from the job in the tree if it is
# unique, otherwise we might unset an attribute we
# have overloaded.
- frozen_job.inheritFrom(job)
+ frozen_job.inheritFrom(job, 'tree job while freezing')
parent.job_trees.append(frozen_tree)
self._createJobTree(change, tree.job_trees, frozen_tree)