Merge "Support custom playbook locations" into feature/zuulv3
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index 1aee1fa..2909ea6 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -26,6 +26,8 @@
[executor]
default_username=zuul
+trusted_ro_dirs=/opt/zuul-scripts:/var/cache
+trusted_rw_dirs=/opt/zuul-logs
[webapp]
listen_address=0.0.0.0
diff --git a/tests/base.py b/tests/base.py
index d89469f..ff1f531 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -933,7 +933,8 @@
'full_name': pr.project
}
},
- 'files': pr.files
+ 'files': pr.files,
+ 'labels': pr.labels
}
return data
@@ -2021,6 +2022,8 @@
project = reponame.replace('_', '/')
self.copyDirToRepo(project,
os.path.join(git_path, reponame))
+ # Make test_root persist after ansible run for .flag test
+ self.config.set('executor', 'trusted_rw_dirs', self.test_root)
self.setupAllProjectKeys()
def setupSimpleLayout(self):
diff --git a/tests/fixtures/layouts/requirements-github.yaml b/tests/fixtures/layouts/requirements-github.yaml
index 9933f27..891a366 100644
--- a/tests/fixtures/layouts/requirements-github.yaml
+++ b/tests/fixtures/layouts/requirements-github.yaml
@@ -168,6 +168,21 @@
github:
comment: true
+- pipeline:
+ name: require_label
+ manager: independent
+ require:
+ github:
+ label: approved
+ trigger:
+ github:
+ - event: pull_request
+ action: comment
+ comment: 'test me'
+ success:
+ github:
+ comment: true
+
- job:
name: project1-pipeline
- job:
@@ -186,6 +201,8 @@
name: project8-requireopen
- job:
name: project9-requirecurrent
+- job:
+ name: project10-label
- project:
name: org/project1
@@ -243,3 +260,9 @@
require_current:
jobs:
- project9-requirecurrent
+
+- project:
+ name: org/project10
+ require_label:
+ jobs:
+ - project10-label
diff --git a/tests/unit/test_bubblewrap.py b/tests/unit/test_bubblewrap.py
index b274944..675221e 100644
--- a/tests/unit/test_bubblewrap.py
+++ b/tests/unit/test_bubblewrap.py
@@ -15,6 +15,7 @@
import subprocess
import tempfile
import testtools
+import os
from zuul.driver import bubblewrap
from zuul.executor.server import SshAgent
@@ -30,17 +31,14 @@
def test_bubblewrap_wraps(self):
bwrap = bubblewrap.BubblewrapDriver()
work_dir = tempfile.mkdtemp()
- ansible_dir = tempfile.mkdtemp()
ssh_agent = SshAgent()
self.addCleanup(ssh_agent.stop)
ssh_agent.start()
po = bwrap.getPopen(work_dir=work_dir,
- ansible_dir=ansible_dir,
ssh_auth_sock=ssh_agent.env['SSH_AUTH_SOCK'])
self.assertTrue(po.passwd_r > 2)
self.assertTrue(po.group_r > 2)
self.assertTrue(work_dir in po.command)
- self.assertTrue(ansible_dir in po.command)
# Now run /usr/bin/id to verify passwd/group entries made it in
true_proc = po(['/usr/bin/id'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
@@ -52,3 +50,23 @@
# Make sure the _r's are closed
self.assertIsNone(po.passwd_r)
self.assertIsNone(po.group_r)
+
+ def test_bubblewrap_leak(self):
+ bwrap = bubblewrap.BubblewrapDriver()
+ work_dir = tempfile.mkdtemp()
+ ansible_dir = tempfile.mkdtemp()
+ ssh_agent = SshAgent()
+ self.addCleanup(ssh_agent.stop)
+ ssh_agent.start()
+ po = bwrap.getPopen(work_dir=work_dir,
+ ansible_dir=ansible_dir,
+ ssh_auth_sock=ssh_agent.env['SSH_AUTH_SOCK'])
+ leak_time = 7
+ # Use hexadecimal notation to avoid false-positive
+ true_proc = po(['bash', '-c', 'sleep 0x%X & disown' % leak_time])
+ self.assertEqual(0, true_proc.wait())
+ cmdline = "sleep\x000x%X\x00" % leak_time
+ sleep_proc = [pid for pid in os.listdir("/proc") if
+ os.path.isfile("/proc/%s/cmdline" % pid) and
+ open("/proc/%s/cmdline" % pid).read() == cmdline]
+ self.assertEqual(len(sleep_proc), 0, "Processes leaked")
diff --git a/tests/unit/test_github_requirements.py b/tests/unit/test_github_requirements.py
index 43bdfc2..135f7ab 100644
--- a/tests/unit/test_github_requirements.py
+++ b/tests/unit/test_github_requirements.py
@@ -350,3 +350,28 @@
self.waitUntilSettled()
# Event hash is not current, should not trigger
self.assertEqual(len(self.history), 1)
+
+ @simple_layout('layouts/requirements-github.yaml', driver='github')
+ def test_pipeline_require_label(self):
+ "Test pipeline requirement: label"
+ A = self.fake_github.openFakePullRequest('org/project10', 'master',
+ 'A')
+ # A comment event that we will keep submitting to trigger
+ comment = A.getCommentAddedEvent('test me')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ # No label so should not be enqueued
+ self.assertEqual(len(self.history), 0)
+
+ # A derp label should not cause it to be enqueued
+ A.addLabel('derp')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # An approved label goes in
+ A.addLabel('approved')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual(self.history[0].name, 'project10-label')
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
old mode 100644
new mode 100755
index 94414f2..dec15e7
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -25,6 +25,7 @@
import zuul.rpcclient
import zuul.cmd
+from zuul.lib.config import get_default
class Client(zuul.cmd.ZuulApp):
@@ -122,22 +123,10 @@
self.setup_logging()
self.server = self.config.get('gearman', 'server')
- if self.config.has_option('gearman', 'port'):
- self.port = self.config.get('gearman', 'port')
- else:
- self.port = 4730
- if self.config.has_option('gearman', 'ssl_key'):
- self.ssl_key = self.config.get('gearman', 'ssl_key')
- else:
- self.ssl_key = None
- if self.config.has_option('gearman', 'ssl_cert'):
- self.ssl_cert = self.config.get('gearman', 'ssl_cert')
- else:
- self.ssl_cert = None
- if self.config.has_option('gearman', 'ssl_ca'):
- self.ssl_ca = self.config.get('gearman', 'ssl_ca')
- else:
- self.ssl_ca = None
+ self.port = get_default(self.config, 'gearman', 'port', 4730)
+ self.ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+ self.ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+ self.ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
if self.args.func():
sys.exit(0)
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 7cc8dd8..57ecfa3 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -32,6 +32,7 @@
import zuul.cmd
import zuul.executor.server
+from zuul.lib.config import get_default
# No zuul imports that pull in paramiko here; it must not be
# imported until after the daemonization.
@@ -63,11 +64,8 @@
self.args = parser.parse_args()
def send_command(self, cmd):
- if self.config.has_option('zuul', 'state_dir'):
- state_dir = os.path.expanduser(
- self.config.get('zuul', 'state_dir'))
- else:
- state_dir = '/var/lib/zuul'
+ state_dir = get_default(self.config, 'zuul', 'state_dir',
+ '/var/lib/zuul', expand_user=True)
path = os.path.join(state_dir, 'executor.socket')
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(path)
@@ -114,10 +112,7 @@
def main(self, daemon=True):
# See comment at top of file about zuul imports
- if self.config.has_option('executor', 'user'):
- self.user = self.config.get('executor', 'user')
- else:
- self.user = 'zuul'
+ self.user = get_default(self.config, 'executor', 'user', 'zuul')
if self.config.has_option('zuul', 'jobroot_dir'):
self.jobroot_dir = os.path.expanduser(
@@ -132,10 +127,8 @@
self.setup_logging('executor', 'log_config')
self.log = logging.getLogger("zuul.Executor")
- if self.config.has_option('executor', 'finger_port'):
- self.finger_port = int(self.config.get('executor', 'finger_port'))
- else:
- self.finger_port = DEFAULT_FINGER_PORT
+ self.finger_port = int(get_default(self.config, 'executor',
+ 'finger_port', DEFAULT_FINGER_PORT))
self.start_log_streamer()
self.change_privs()
@@ -170,10 +163,9 @@
server.configure_connections(source_only=True)
- if server.config.has_option('executor', 'pidfile'):
- pid_fn = os.path.expanduser(server.config.get('executor', 'pidfile'))
- else:
- pid_fn = '/var/run/zuul-executor/zuul-executor.pid'
+ pid_fn = get_default(server.config, 'executor', 'pidfile',
+ '/var/run/zuul-executor/zuul-executor.pid',
+ expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if server.args.nodaemon:
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
index 686f34a..97f208c 100755
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -27,6 +27,7 @@
import signal
import zuul.cmd
+from zuul.lib.config import get_default
# No zuul imports here because they pull in paramiko which must not be
# imported until after the daemonization.
@@ -79,10 +80,8 @@
server.read_config()
server.configure_connections(source_only=True)
- if server.config.has_option('zuul', 'state_dir'):
- state_dir = os.path.expanduser(server.config.get('zuul', 'state_dir'))
- else:
- state_dir = '/var/lib/zuul'
+ state_dir = get_default(server.config, 'zuul', 'state_dir',
+ '/var/lib/zuul', expand_user=True)
test_fn = os.path.join(state_dir, 'test')
try:
f = open(test_fn, 'w')
@@ -92,10 +91,9 @@
print("\nUnable to write to state directory: %s\n" % state_dir)
raise
- if server.config.has_option('merger', 'pidfile'):
- pid_fn = os.path.expanduser(server.config.get('merger', 'pidfile'))
- else:
- pid_fn = '/var/run/zuul-merger/zuul-merger.pid'
+ pid_fn = get_default(server.config, 'merger', 'pidfile',
+ '/var/run/zuul-merger/zuul-merger.pid',
+ expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if server.args.nodaemon:
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index d16eb17..b32deaf 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -28,6 +28,7 @@
import signal
import zuul.cmd
+from zuul.lib.config import get_default
# No zuul imports here because they pull in paramiko which must not be
# imported until after the daemonization.
@@ -98,22 +99,10 @@
import zuul.lib.gearserver
statsd_host = os.environ.get('STATSD_HOST')
statsd_port = int(os.environ.get('STATSD_PORT', 8125))
- if self.config.has_option('gearman_server', 'listen_address'):
- host = self.config.get('gearman_server', 'listen_address')
- else:
- host = None
- if self.config.has_option('gearman_server', 'ssl_key'):
- ssl_key = self.config.get('gearman_server', 'ssl_key')
- else:
- ssl_key = None
- if self.config.has_option('gearman_server', 'ssl_cert'):
- ssl_cert = self.config.get('gearman_server', 'ssl_cert')
- else:
- ssl_cert = None
- if self.config.has_option('gearman_server', 'ssl_ca'):
- ssl_ca = self.config.get('gearman_server', 'ssl_ca')
- else:
- ssl_ca = None
+ host = get_default(self.config, 'gearman_server', 'listen_address')
+ ssl_key = get_default(self.config, 'gearman_server', 'ssl_key')
+ ssl_cert = get_default(self.config, 'gearman_server', 'ssl_cert')
+ ssl_ca = get_default(self.config, 'gearman_server', 'ssl_ca')
zuul.lib.gearserver.GearServer(4730,
ssl_key=ssl_key,
ssl_cert=ssl_cert,
@@ -161,27 +150,16 @@
nodepool = zuul.nodepool.Nodepool(self.sched)
zookeeper = zuul.zk.ZooKeeper()
- if self.config.has_option('zuul', 'zookeeper_hosts'):
- zookeeper_hosts = self.config.get('zuul', 'zookeeper_hosts')
- else:
- zookeeper_hosts = '127.0.0.1:2181'
+ zookeeper_hosts = get_default(self.config, 'zuul', 'zookeeper_hosts',
+ '127.0.0.1:2181')
zookeeper.connect(zookeeper_hosts)
- if self.config.has_option('zuul', 'status_expiry'):
- cache_expiry = self.config.getint('zuul', 'status_expiry')
- else:
- cache_expiry = 1
+ cache_expiry = get_default(self.config, 'zuul', 'status_expiry', 1)
- if self.config.has_option('webapp', 'listen_address'):
- listen_address = self.config.get('webapp', 'listen_address')
- else:
- listen_address = '0.0.0.0'
-
- if self.config.has_option('webapp', 'port'):
- port = self.config.getint('webapp', 'port')
- else:
- port = 8001
+ listen_address = get_default(self.config, 'webapp', 'listen_address',
+ '0.0.0.0')
+ port = get_default(self.config, 'webapp', 'port', 8001)
webapp = zuul.webapp.WebApp(
self.sched, port=port, cache_expiry=cache_expiry,
@@ -230,10 +208,9 @@
if scheduler.args.validate:
sys.exit(scheduler.test_config())
- if scheduler.config.has_option('zuul', 'pidfile'):
- pid_fn = os.path.expanduser(scheduler.config.get('zuul', 'pidfile'))
- else:
- pid_fn = '/var/run/zuul-scheduler/zuul-scheduler.pid'
+ pid_fn = get_default(scheduler.config, 'zuul', 'pidfile',
+ '/var/run/zuul-scheduler/zuul-scheduler.pid',
+ expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if scheduler.args.nodaemon:
diff --git a/zuul/driver/__init__.py b/zuul/driver/__init__.py
index b6c34e7..c78283d 100644
--- a/zuul/driver/__init__.py
+++ b/zuul/driver/__init__.py
@@ -270,3 +270,13 @@
:rtype: Callable
"""
pass
+
+ @abc.abstractmethod
+ def setMountsMap(self, state_dir, ro_dirs=[], rw_dirs=[]):
+ """Add additional mount point to the execution environment.
+
+ :arg str state_dir: the state directory to be read write
+ :arg list ro_dirs: read only directories paths
+ :arg list rw_dirs: read write directories paths
+ """
+ pass
diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py
index c165727..9e9a26e 100644
--- a/zuul/driver/bubblewrap/__init__.py
+++ b/zuul/driver/bubblewrap/__init__.py
@@ -83,7 +83,6 @@
'--ro-bind', '/bin', '/bin',
'--ro-bind', '/sbin', '/sbin',
'--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf',
- '--ro-bind', '{ansible_dir}', '{ansible_dir}',
'--ro-bind', '{ssh_auth_sock}', '{ssh_auth_sock}',
'--dir', '{work_dir}',
'--bind', '{work_dir}', '{work_dir}',
@@ -92,11 +91,13 @@
'--chdir', '{work_dir}',
'--unshare-all',
'--share-net',
+ '--die-with-parent',
'--uid', '{uid}',
'--gid', '{gid}',
'--file', '{uid_fd}', '/etc/passwd',
'--file', '{gid_fd}', '/etc/group',
]
+ mounts_map = {'rw': [], 'ro': []}
def reconfigure(self, tenant):
pass
@@ -104,6 +105,9 @@
def stop(self):
pass
+ def setMountsMap(self, state_dir, ro_dirs=[], rw_dirs=[]):
+ self.mounts_map = {'ro': ro_dirs, 'rw': [state_dir] + rw_dirs}
+
def getPopen(self, **kwargs):
# Set zuul_dir if it was not passed in
if 'zuul_dir' in kwargs:
@@ -117,6 +121,11 @@
if not zuul_dir.startswith('/usr'):
bwrap_command.extend(['--ro-bind', zuul_dir, zuul_dir])
+ for mount_type in ('ro', 'rw'):
+ bind_arg = '--ro-bind' if mount_type == 'ro' else '--bind'
+ for bind in self.mounts_map[mount_type]:
+ bwrap_command.extend([bind_arg, bind, bind])
+
# Need users and groups
uid = os.getuid()
passwd = pwd.getpwuid(uid)
@@ -159,14 +168,12 @@
parser = argparse.ArgumentParser()
parser.add_argument('work_dir')
- parser.add_argument('ansible_dir')
parser.add_argument('run_args', nargs='+')
cli_args = parser.parse_args()
ssh_auth_sock = os.environ.get('SSH_AUTH_SOCK')
popen = driver.getPopen(work_dir=cli_args.work_dir,
- ansible_dir=cli_args.ansible_dir,
ssh_auth_sock=ssh_auth_sock)
x = popen(cli_args.run_args)
x.wait()
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index f9f1c27..7a3491e 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -524,6 +524,7 @@
change.patchset)
change.reviews = self.getPullReviews(change.project,
change.number)
+ change.labels = change.pr.get('labels')
return change
@@ -572,8 +573,11 @@
self.log.warning("Pull request #%s of %s/%s returned None!" % (
number, owner, proj))
time.sleep(1)
+ # Get the issue obj so we can get the labels (this is silly)
+ issueobj = probj.issue()
pr = probj.as_dict()
pr['files'] = [f.filename for f in probj.files()]
+ pr['labels'] = [l.name for l in issueobj.labels()]
log_rate_limit(self.log, github)
return pr
@@ -599,7 +603,7 @@
if not pr_url:
continue
# the issue provides no good description of the project :\
- owner, project, _, number = pr_url.split('/')[4:]
+ owner, project, _, number = pr_url.split('/')[-4:]
github = self.getGithubClient("%s/%s" % (owner, project))
pr = github.pull_request(owner, project, number)
if pr.head.sha != sha:
diff --git a/zuul/driver/github/githubmodel.py b/zuul/driver/github/githubmodel.py
index cfd1bc0..db119f0 100644
--- a/zuul/driver/github/githubmodel.py
+++ b/zuul/driver/github/githubmodel.py
@@ -28,9 +28,13 @@
class PullRequest(Change):
def __init__(self, project):
super(PullRequest, self).__init__(project)
+ self.project = None
+ self.pr = None
self.updated_at = None
self.title = None
self.reviews = []
+ self.files = []
+ self.labels = []
def isUpdateOf(self, other):
if (hasattr(other, 'number') and self.number == other.number and
@@ -284,7 +288,8 @@
class GithubRefFilter(RefFilter, GithubCommonFilter):
def __init__(self, connection_name, statuses=[], required_reviews=[],
- reject_reviews=[], open=None, current_patchset=None):
+ reject_reviews=[], open=None, current_patchset=None,
+ labels=[]):
RefFilter.__init__(self, connection_name)
GithubCommonFilter.__init__(self, required_reviews=required_reviews,
@@ -293,6 +298,7 @@
self.statuses = statuses
self.open = open
self.current_patchset = current_patchset
+ self.labels = labels
def __repr__(self):
ret = '<GithubRefFilter'
@@ -310,6 +316,8 @@
ret += ' open: %s' % self.open
if self.current_patchset:
ret += ' current-patchset: %s' % self.current_patchset
+ if self.labels:
+ ret += ' labels: %s' % self.labels
ret += '>'
@@ -341,4 +349,9 @@
if not self.matchesReviews(change):
return False
+ # required labels are ANDed
+ for label in self.labels:
+ if label not in change.labels:
+ return False
+
return True
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
index 519ebf1..1bd280f 100644
--- a/zuul/driver/github/githubsource.py
+++ b/zuul/driver/github/githubsource.py
@@ -97,6 +97,7 @@
required_reviews=to_list(config.get('review')),
open=config.get('open'),
current_patchset=config.get('current-patchset'),
+ labels=to_list(config.get('label')),
)
return [f]
@@ -121,7 +122,8 @@
require = {'status': scalar_or_list(str),
'review': scalar_or_list(review),
'open': bool,
- 'current-patchset': bool}
+ 'current-patchset': bool,
+ 'label': scalar_or_list(str)}
return require
diff --git a/zuul/driver/nullwrap/__init__.py b/zuul/driver/nullwrap/__init__.py
index ebcd1da..50fea27 100644
--- a/zuul/driver/nullwrap/__init__.py
+++ b/zuul/driver/nullwrap/__init__.py
@@ -26,3 +26,6 @@
def getPopen(self, **kwargs):
return subprocess.Popen
+
+ def setMountsMap(self, **kwargs):
+ pass
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 6ecb27c..2e17b3e 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -21,6 +21,7 @@
from uuid import uuid4
import zuul.model
+from zuul.lib.config import get_default
from zuul.model import Build
@@ -115,23 +116,10 @@
self.meta_jobs = {} # A list of meta-jobs like stop or describe
server = config.get('gearman', 'server')
- if config.has_option('gearman', 'port'):
- port = config.get('gearman', 'port')
- else:
- port = 4730
- if self.config.has_option('gearman', 'ssl_key'):
- ssl_key = self.config.get('gearman', 'ssl_key')
- else:
- ssl_key = None
- if self.config.has_option('gearman', 'ssl_cert'):
- ssl_cert = self.config.get('gearman', 'ssl_cert')
- else:
- ssl_cert = None
- if self.config.has_option('gearman', 'ssl_ca'):
- ssl_ca = self.config.get('gearman', 'ssl_ca')
- else:
- ssl_ca = None
-
+ port = get_default(self.config, 'gearman', 'port', 4730)
+ ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+ ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+ ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
self.gearman = ZuulGearmanClient(self)
self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 94678d7..657a063 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -26,6 +26,7 @@
import time
import traceback
from zuul.lib.yamlutil import yaml
+from zuul.lib.config import get_default
import gear
@@ -375,33 +376,15 @@
unverbose=self.verboseOff,
)
- if self.config.has_option('executor', 'git_dir'):
- self.merge_root = self.config.get('executor', 'git_dir')
- else:
- self.merge_root = '/var/lib/zuul/executor-git'
-
- if self.config.has_option('executor', 'default_username'):
- self.default_username = self.config.get('executor',
- 'default_username')
- else:
- self.default_username = 'zuul'
-
- if self.config.has_option('merger', 'git_user_email'):
- self.merge_email = self.config.get('merger', 'git_user_email')
- else:
- self.merge_email = None
-
- if self.config.has_option('merger', 'git_user_name'):
- self.merge_name = self.config.get('merger', 'git_user_name')
- else:
- self.merge_name = None
-
- if self.config.has_option('executor', 'untrusted_wrapper'):
- untrusted_wrapper_name = self.config.get(
- 'executor', 'untrusted_wrapper').strip()
- else:
- untrusted_wrapper_name = 'bubblewrap'
- self.untrusted_wrapper = connections.drivers[untrusted_wrapper_name]
+ self.merge_root = get_default(self.config, 'executor', 'git_dir',
+ '/var/lib/zuul/executor-git')
+ self.default_username = get_default(self.config, 'executor',
+ 'default_username', 'zuul')
+ self.merge_email = get_default(self.config, 'merger', 'git_user_email')
+ self.merge_name = get_default(self.config, 'merger', 'git_user_name')
+ execution_wrapper_name = get_default(self.config, 'executor',
+ 'execution_wrapper', 'bubblewrap')
+ self.execution_wrapper = connections.drivers[execution_wrapper_name]
self.connections = connections
# This merger and its git repos are used to maintain
@@ -411,11 +394,8 @@
self.merger = self._getMerger(self.merge_root)
self.update_queue = DeduplicateQueue()
- if self.config.has_option('zuul', 'state_dir'):
- state_dir = os.path.expanduser(
- self.config.get('zuul', 'state_dir'))
- else:
- state_dir = '/var/lib/zuul'
+ state_dir = get_default(self.config, 'zuul', 'state_dir',
+ '/var/lib/zuul', expand_user=True)
path = os.path.join(state_dir, 'executor.socket')
self.command_socket = commandsocket.CommandSocket(path)
ansible_dir = os.path.join(state_dir, 'ansible')
@@ -457,22 +437,10 @@
self._running = True
self._command_running = True
server = self.config.get('gearman', 'server')
- if self.config.has_option('gearman', 'port'):
- port = self.config.get('gearman', 'port')
- else:
- port = 4730
- if self.config.has_option('gearman', 'ssl_key'):
- ssl_key = self.config.get('gearman', 'ssl_key')
- else:
- ssl_key = None
- if self.config.has_option('gearman', 'ssl_cert'):
- ssl_cert = self.config.get('gearman', 'ssl_cert')
- else:
- ssl_cert = None
- if self.config.has_option('gearman', 'ssl_ca'):
- ssl_ca = self.config.get('gearman', 'ssl_ca')
- else:
- ssl_ca = None
+ port = get_default(self.config, 'gearman', 'port', 4730)
+ ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+ ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+ ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
self.merger_worker = ExecutorMergeWorker(self, 'Zuul Executor Merger')
self.merger_worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
self.executor_worker = gear.TextWorker('Zuul Executor Server')
@@ -715,12 +683,9 @@
self.thread = None
self.ssh_agent = None
- if self.executor_server.config.has_option(
- 'executor', 'private_key_file'):
- self.private_key_file = self.executor_server.config.get(
- 'executor', 'private_key_file')
- else:
- self.private_key_file = '~/.ssh/id_rsa'
+ self.private_key_file = get_default(self.executor_server.config,
+ 'executor', 'private_key_file',
+ '~/.ssh/id_rsa')
self.ssh_agent = SshAgent()
def run(self):
@@ -1273,14 +1238,24 @@
if trusted:
config_file = self.jobdir.trusted_config
- popen = subprocess.Popen
+ opt_prefix = 'trusted'
else:
config_file = self.jobdir.untrusted_config
- driver = self.executor_server.untrusted_wrapper
- popen = driver.getPopen(
- work_dir=self.jobdir.root,
- ansible_dir=self.executor_server.ansible_dir,
- ssh_auth_sock=env_copy.get('SSH_AUTH_SOCK'))
+ opt_prefix = 'untrusted'
+ ro_dirs = get_default(self.executor_server.config, 'executor',
+ '%s_ro_dirs' % opt_prefix)
+ rw_dirs = get_default(self.executor_server.config, 'executor',
+ '%s_rw_dirs' % opt_prefix)
+ state_dir = get_default(self.executor_server.config, 'zuul',
+ 'state_dir', '/var/lib/zuul', expand_user=True)
+ ro_dirs = ro_dirs.split(":") if ro_dirs else []
+ rw_dirs = rw_dirs.split(":") if rw_dirs else []
+ self.executor_server.execution_wrapper.setMountsMap(state_dir, ro_dirs,
+ rw_dirs)
+
+ popen = self.executor_server.execution_wrapper.getPopen(
+ work_dir=self.jobdir.root,
+ ssh_auth_sock=env_copy.get('SSH_AUTH_SOCK'))
env_copy['ANSIBLE_CONFIG'] = config_file
diff --git a/zuul/lib/config.py b/zuul/lib/config.py
new file mode 100644
index 0000000..9cdf66e
--- /dev/null
+++ b/zuul/lib/config.py
@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+
+def get_default(config, section, option, default=None, expand_user=False):
+ if config.has_option(section, option):
+ value = config.get(section, option)
+ else:
+ value = default
+ if expand_user and value:
+ return os.path.expanduser(value)
+ return value
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 4054df6..e92d9fd 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -20,6 +20,7 @@
import gear
import zuul.model
+from zuul.lib.config import get_default
def getJobData(job):
@@ -75,22 +76,10 @@
self.config = config
self.sched = sched
server = self.config.get('gearman', 'server')
- if self.config.has_option('gearman', 'port'):
- port = self.config.get('gearman', 'port')
- else:
- port = 4730
- if self.config.has_option('gearman', 'ssl_key'):
- ssl_key = self.config.get('gearman', 'ssl_key')
- else:
- ssl_key = None
- if self.config.has_option('gearman', 'ssl_cert'):
- ssl_cert = self.config.get('gearman', 'ssl_cert')
- else:
- ssl_cert = None
- if self.config.has_option('gearman', 'ssl_ca'):
- ssl_ca = self.config.get('gearman', 'ssl_ca')
- else:
- ssl_ca = None
+ port = get_default(self.config, 'gearman', 'port', 4730)
+ ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+ ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+ ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
self.log.debug("Connecting to gearman at %s:%s" % (server, port))
self.gearman = MergeGearmanClient(self)
self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 7d7e771..cbc4cb8 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -19,6 +19,7 @@
import gear
+from zuul.lib.config import get_default
from zuul.merger import merger
@@ -29,20 +30,10 @@
self.config = config
self.zuul_url = config.get('merger', 'zuul_url')
- if self.config.has_option('merger', 'git_dir'):
- merge_root = self.config.get('merger', 'git_dir')
- else:
- merge_root = '/var/lib/zuul/merger-git'
-
- if self.config.has_option('merger', 'git_user_email'):
- merge_email = self.config.get('merger', 'git_user_email')
- else:
- merge_email = None
-
- if self.config.has_option('merger', 'git_user_name'):
- merge_name = self.config.get('merger', 'git_user_name')
- else:
- merge_name = None
+ merge_root = get_default(self.config, 'merger', 'git_dir',
+ '/var/lib/zuul/merger-git')
+ merge_email = get_default(self.config, 'merger', 'git_user_email')
+ merge_name = get_default(self.config, 'merger', 'git_user_name')
self.merger = merger.Merger(merge_root, connections, merge_email,
merge_name)
@@ -50,22 +41,10 @@
def start(self):
self._running = True
server = self.config.get('gearman', 'server')
- if self.config.has_option('gearman', 'port'):
- port = self.config.get('gearman', 'port')
- else:
- port = 4730
- if self.config.has_option('gearman', 'ssl_key'):
- ssl_key = self.config.get('gearman', 'ssl_key')
- else:
- ssl_key = None
- if self.config.has_option('gearman', 'ssl_cert'):
- ssl_cert = self.config.get('gearman', 'ssl_cert')
- else:
- ssl_cert = None
- if self.config.has_option('gearman', 'ssl_ca'):
- ssl_ca = self.config.get('gearman', 'ssl_ca')
- else:
- ssl_ca = None
+ port = get_default(self.config, 'gearman', 'port', 4730)
+ ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+ ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+ ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
self.worker = gear.TextWorker('Zuul Merger')
self.worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
self.log.debug("Waiting for server")
diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py
index c1ee50e..be3b7d1 100644
--- a/zuul/rpclistener.py
+++ b/zuul/rpclistener.py
@@ -21,6 +21,7 @@
import gear
from zuul import model
+from zuul.lib.config import get_default
class RPCListener(object):
@@ -33,22 +34,10 @@
def start(self):
self._running = True
server = self.config.get('gearman', 'server')
- if self.config.has_option('gearman', 'port'):
- port = self.config.get('gearman', 'port')
- else:
- port = 4730
- if self.config.has_option('gearman', 'ssl_key'):
- ssl_key = self.config.get('gearman', 'ssl_key')
- else:
- ssl_key = None
- if self.config.has_option('gearman', 'ssl_cert'):
- ssl_cert = self.config.get('gearman', 'ssl_cert')
- else:
- ssl_cert = None
- if self.config.has_option('gearman', 'ssl_ca'):
- ssl_ca = self.config.get('gearman', 'ssl_ca')
- else:
- ssl_ca = None
+ port = get_default(self.config, 'gearman', 'port', 4730)
+ ssl_key = get_default(self.config, 'gearman', 'ssl_key')
+ ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
+ ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
self.worker = gear.TextWorker('Zuul RPC Listener')
self.worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
self.log.debug("Waiting for server")
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 2076163..c762309 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -30,6 +30,7 @@
from zuul import model
from zuul import exceptions
from zuul import version as zuul_version
+from zuul.lib.config import get_default
class ManagementEvent(object):
@@ -371,30 +372,21 @@
self.log.debug("Waiting for exit")
def _get_queue_pickle_file(self):
- if self.config.has_option('zuul', 'state_dir'):
- state_dir = os.path.expanduser(self.config.get('zuul',
- 'state_dir'))
- else:
- state_dir = '/var/lib/zuul'
+ state_dir = get_default(self.config, 'zuul', 'state_dir',
+ '/var/lib/zuul', expand_user=True)
return os.path.join(state_dir, 'queue.pickle')
def _get_time_database_dir(self):
- if self.config.has_option('zuul', 'state_dir'):
- state_dir = os.path.expanduser(self.config.get('zuul',
- 'state_dir'))
- else:
- state_dir = '/var/lib/zuul'
+ state_dir = get_default(self.config, 'zuul', 'state_dir',
+ '/var/lib/zuul', expand_user=True)
d = os.path.join(state_dir, 'times')
if not os.path.exists(d):
os.mkdir(d)
return d
def _get_project_key_dir(self):
- if self.config.has_option('zuul', 'state_dir'):
- state_dir = os.path.expanduser(self.config.get('zuul',
- 'state_dir'))
- else:
- state_dir = '/var/lib/zuul'
+ state_dir = get_default(self.config, 'zuul', 'state_dir',
+ '/var/lib/zuul', expand_user=True)
key_dir = os.path.join(state_dir, 'keys')
if not os.path.exists(key_dir):
os.mkdir(key_dir, 0o700)