Merge "Allow file manipulation in the work dir" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index d3be81b..e8b070f 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -7,8 +7,3 @@
- tox-linters
- tox-py35
- tox-tarball
- - zuul-tox-docs
- - zuul-tox-cover
- - zuul-tox-linters
- - zuul-tox-py35
- - zuul-tox-tarball
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index b91e7e7..a24b833 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -54,18 +54,14 @@
zookeeper
"""""""""
+.. NOTE: this is a white lie at this point, since only the scheduler
+ uses this, however, we expect other components to use it later, so
+ it's reasonable for admins to plan for this now.
+
**hosts**
A list of zookeeper hosts for Zuul to use when communicating with
Nodepool. ``hosts=zk1.example.com,zk2.example.com,zk3.example.com``
-zuul
-""""
-
-**status_url**
- URL that will be posted in Zuul comments made to changes when
- starting jobs for a change. Used by zuul-scheduler only.
- ``status_url=https://zuul.example.com/status``
-
Scheduler
---------
@@ -126,6 +122,11 @@
optional value and ``1`` is used by default.
``status_expiry=1``
+**status_url**
+ URL that will be posted in Zuul comments made to changes when
+ starting jobs for a change. Used by zuul-scheduler only.
+ ``status_url=https://zuul.example.com/status``
+
scheduler
"""""""""
diff --git a/doc/source/admin/drivers/gerrit.rst b/doc/source/admin/drivers/gerrit.rst
index 470b4e8..29e136b 100644
--- a/doc/source/admin/drivers/gerrit.rst
+++ b/doc/source/admin/drivers/gerrit.rst
@@ -35,12 +35,12 @@
**canonical_hostname**
The canonical hostname associated with the git repos on the Gerrit
server. Defaults to the value of **server**. This is used to
- identify repos from this connection by name and in preparing repos
- on the filesystem for use by jobs. This only needs to be set in the
- case where the canonical public location of the git repos is not the
- same as the Gerrit server and it would be incorrect to refer to
- those repos in configuration and build scripts using the Gerrit
- server hostname.
+ identify projects from this connection by name and in preparing
+ repos on the filesystem for use by jobs. Note that Zuul will still
+ only communicate with the Gerrit server identified by **server**;
+ this option is useful if users customarily use a different hostname
+ to clone or pull git repos so that when Zuul places them in the
+ job's working directory, they appear under this directory name.
``canonical_hostname=git.example.com``
**port**
diff --git a/doc/source/admin/drivers/github.rst b/doc/source/admin/drivers/github.rst
index 0cbf895..9740292 100644
--- a/doc/source/admin/drivers/github.rst
+++ b/doc/source/admin/drivers/github.rst
@@ -41,20 +41,20 @@
Path to SSH key to use when cloning github repositories.
``sshkey=/home/zuul/.ssh/id_rsa``
-**git_host**
+**server**
Optional: Hostname of the github install (such as a GitHub Enterprise)
If not specified, defaults to ``github.com``
- ``git_host=github.myenterprise.com``
+ ``server=github.myenterprise.com``
**canonical_hostname**
The canonical hostname associated with the git repos on the GitHub
- server. Defaults to the value of **git_host**. This is used to
- identify repos from this connection by name and in preparing repos
- on the filesystem for use by jobs. This only needs to be set in the
- case where the canonical public location of the git repos is not the
- same as the GitHub server and it would be incorrect to refer to
- those repos in configuration and build scripts using the GitHub
- server hostname.
+ server. Defaults to the value of **server**. This is used to
+ identify projects from this connection by name and in preparing
+ repos on the filesystem for use by jobs. Note that Zuul will still
+ only communicate with the GitHub server identified by **server**;
+ this option is useful if users customarily use a different hostname
+ to clone or pull git repos so that when Zuul places them in the
+ job's working directory, they appear under this directory name.
``canonical_hostname=git.example.com``
Trigger Configuration
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index e7226e9..0b2b5d4 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -99,7 +99,7 @@
+1, or if at least one of them fails, a -1::
- pipeline:
- name: check
+ name: check
manager: independent
trigger:
my_gerrit:
@@ -164,6 +164,17 @@
For more detail on the theory and operation of Zuul's dependent
pipeline manager, see: :doc:`gating`.
+**allow-secrets**
+ This is a boolean which can be used to prevent jobs which require
+ secrets from running in this pipeline. Some pipelines run on
+ proposed changes and therefore execute code which has not yet been
+ reviewed. In such a case, allowing a job to use a secret could
+ result in that secret being exposed. The default is False, meaning
+ that in order to run jobs with secrets, this must be explicitly
+ enabled on each Pipeline where that is safe.
+
+ For more information, see :ref:`secret`.
+
**description**
This field may be used to provide a textual description of the
pipeline. It may appear in the status page or in documentation.
diff --git a/tests/base.py b/tests/base.py
index 617169a..921fcd1 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -554,7 +554,7 @@
def __init__(self, github, number, project, branch,
subject, upstream_root, files=[], number_of_commits=1,
- writers=[], body=''):
+ writers=[], body=None):
"""Creates a new PR with several commits.
Sends an event about opened PR."""
self.github = github
@@ -880,7 +880,7 @@
self.reports = []
def openFakePullRequest(self, project, branch, subject, files=[],
- body=''):
+ body=None):
self.pr_number += 1
pull_request = FakeGithubPullRequest(
self, self.pr_number, project, branch, subject, self.upstream_root,
@@ -922,7 +922,7 @@
'http://localhost:%s/connection/%s/payload'
% (port, self.connection_name),
data=payload, headers=headers)
- urllib.request.urlopen(req)
+ return urllib.request.urlopen(req)
def getPull(self, project, number):
pr = self.pull_requests[number - 1]
@@ -1048,10 +1048,14 @@
def _getNeededByFromPR(self, change):
prs = []
pattern = re.compile(r"Depends-On.*https://%s/%s/pull/%s" %
- (self.git_host, change.project.name,
+ (self.server, change.project.name,
change.number))
for pr in self.pull_requests:
- if pattern.search(pr.body):
+ if not pr.body:
+ body = ''
+ else:
+ body = pr.body
+ if pattern.search(body):
# Get our version of a pull so that it's a dict
pull = self.getPull(pr.project, pr.number)
prs.append(pull)
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/post-broken.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/post-broken.yaml
new file mode 100644
index 0000000..cf61187
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/post-broken.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+ tasks:
+ - shell: |+
+ echo "I am broken"
+ exit 1
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index fd3fc6d..aa57d08 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -48,8 +48,13 @@
Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+- job:
+ name: base-urls
+ success-url: https://success.example.com/zuul-logs/{build.uuid}/
+ failure-url: https://failure.example.com/zuul-logs/{build.uuid}/
- job:
+ parent: base-urls
name: python27
pre-run: playbooks/pre
post-run: playbooks/post
@@ -74,5 +79,11 @@
label: ubuntu-xenial
- job:
+ parent: base-urls
name: hello
post-run: playbooks/hello-post
+
+- job:
+ parent: python27
+ name: failpost
+ post-run: playbooks/post-broken
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
index ca734c5..e87d988 100644
--- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -15,3 +15,4 @@
- check-vars
- timeout
- hello-world
+ - failpost
diff --git a/tests/fixtures/zuul-connections-merger.conf b/tests/fixtures/zuul-connections-merger.conf
index 4499493..df465d5 100644
--- a/tests/fixtures/zuul-connections-merger.conf
+++ b/tests/fixtures/zuul-connections-merger.conf
@@ -1,7 +1,7 @@
[gearman]
server=127.0.0.1
-[zuul]
+[webapp]
status_url=http://zuul.example.com/status
[merger]
diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf
index dc28f98..3d61ab6 100644
--- a/tests/fixtures/zuul-github-driver.conf
+++ b/tests/fixtures/zuul-github-driver.conf
@@ -1,7 +1,7 @@
[gearman]
server=127.0.0.1
-[zuul]
+[webapp]
status_url=http://zuul.example.com/status/#{change.number},{change.patchset}
[merger]
@@ -23,4 +23,4 @@
[connection github_ent]
driver=github
sshkey=/home/zuul/.ssh/id_rsa
-git_host=github.enterprise.io
+server=github.enterprise.io
diff --git a/tests/fixtures/zuul-push-reqs.conf b/tests/fixtures/zuul-push-reqs.conf
index c5272aa..4faac13 100644
--- a/tests/fixtures/zuul-push-reqs.conf
+++ b/tests/fixtures/zuul-push-reqs.conf
@@ -1,7 +1,7 @@
[gearman]
server=127.0.0.1
-[zuul]
+[webapp]
status_url=http://zuul.example.com/status
[merger]
diff --git a/tests/unit/test_bubblewrap.py b/tests/unit/test_bubblewrap.py
index 675221e..d94b3f2 100644
--- a/tests/unit/test_bubblewrap.py
+++ b/tests/unit/test_bubblewrap.py
@@ -15,10 +15,12 @@
import subprocess
import tempfile
import testtools
+import time
import os
from zuul.driver import bubblewrap
from zuul.executor.server import SshAgent
+from tests.base import iterate_timeout
class TestBubblewrap(testtools.TestCase):
@@ -61,12 +63,15 @@
po = bwrap.getPopen(work_dir=work_dir,
ansible_dir=ansible_dir,
ssh_auth_sock=ssh_agent.env['SSH_AUTH_SOCK'])
- leak_time = 7
+ leak_time = 60
# Use hexadecimal notation to avoid false-positive
true_proc = po(['bash', '-c', 'sleep 0x%X & disown' % leak_time])
self.assertEqual(0, true_proc.wait())
cmdline = "sleep\x000x%X\x00" % leak_time
- sleep_proc = [pid for pid in os.listdir("/proc") if
- os.path.isfile("/proc/%s/cmdline" % pid) and
- open("/proc/%s/cmdline" % pid).read() == cmdline]
- self.assertEqual(len(sleep_proc), 0, "Processes leaked")
+ for x in iterate_timeout(30, "process to exit"):
+ sleep_proc = [pid for pid in os.listdir("/proc") if
+ os.path.isfile("/proc/%s/cmdline" % pid) and
+ open("/proc/%s/cmdline" % pid).read() == cmdline]
+ if not sleep_proc:
+ break
+ time.sleep(1)
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index a19073c..f360866 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -14,6 +14,7 @@
import re
from testtools.matchers import MatchesRegex, StartsWith
+import urllib
import time
from tests.base import ZuulTestCase, simple_layout, random_sha1
@@ -584,3 +585,18 @@
new = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
# New timestamp should be greater than the old timestamp
self.assertLess(old, new)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_ping_event(self):
+ # Test valid ping
+ pevent = {'repository': {'full_name': 'org/project'}}
+ req = self.fake_github.emitEvent(('ping', pevent))
+ self.assertEqual(req.status, 200, "Ping event didn't succeed")
+
+ # Test invalid ping
+ pevent = {'repository': {'full_name': 'unknown-project'}}
+ self.assertRaises(
+ urllib.error.HTTPError,
+ self.fake_github.emitEvent,
+ ('ping', pevent),
+ )
diff --git a/tests/unit/test_github_requirements.py b/tests/unit/test_github_requirements.py
index 135f7ab..f125d1e 100644
--- a/tests/unit/test_github_requirements.py
+++ b/tests/unit/test_github_requirements.py
@@ -240,13 +240,10 @@
# The first negative review from derp should not cause it to be
# enqueued
- for i in range(1, 4):
- submitted_at = time.time() - 72 * 60 * 60
- A.addReview('derp', 'CHANGES_REQUESTED',
- submitted_at)
- self.fake_github.emitEvent(comment)
- self.waitUntilSettled()
- self.assertEqual(len(self.history), 0)
+ A.addReview('derp', 'CHANGES_REQUESTED')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
# A positive review from derp should cause it to be enqueued
A.addReview('derp', 'APPROVED')
@@ -256,6 +253,37 @@
self.assertEqual(self.history[0].name, 'project5-reviewuserstate')
@simple_layout('layouts/requirements-github.yaml', driver='github')
+ def test_pipeline_require_review_comment_masked(self):
+ "Test pipeline requirement: review comments on top of votes"
+
+ A = self.fake_github.openFakePullRequest('org/project5', 'master', 'A')
+ # Add derp to writers
+ A.writers.append('derp')
+ # A comment event that we will keep submitting to trigger
+ comment = A.getCommentAddedEvent('test me')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ # No positive review from derp so should not be enqueued
+ self.assertEqual(len(self.history), 0)
+
+ # The first negative review from derp should not cause it to be
+ # enqueued
+ A.addReview('derp', 'CHANGES_REQUESTED')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # A positive review is required, so provide it
+ A.addReview('derp', 'APPROVED')
+
+ # Add a comment review on top to make sure we can still enqueue
+ A.addReview('derp', 'COMMENTED')
+ self.fake_github.emitEvent(comment)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual(self.history[0].name, 'project5-reviewuserstate')
+
+ @simple_layout('layouts/requirements-github.yaml', driver='github')
def test_require_review_newer_than(self):
A = self.fake_github.openFakePullRequest('org/project6', 'master', 'A')
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 7c5fa70..327f745 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -496,39 +496,52 @@
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
- build = self.getJobFromHistory('timeout')
- self.assertEqual(build.result, 'TIMED_OUT')
- build = self.getJobFromHistory('faillocal')
- self.assertEqual(build.result, 'FAILURE')
- build = self.getJobFromHistory('check-vars')
- self.assertEqual(build.result, 'SUCCESS')
- build = self.getJobFromHistory('hello-world')
- self.assertEqual(build.result, 'SUCCESS')
- build = self.getJobFromHistory('python27')
- self.assertEqual(build.result, 'SUCCESS')
- flag_path = os.path.join(self.test_root, build.uuid + '.flag')
+ build_timeout = self.getJobFromHistory('timeout')
+ self.assertEqual(build_timeout.result, 'TIMED_OUT')
+ build_faillocal = self.getJobFromHistory('faillocal')
+ self.assertEqual(build_faillocal.result, 'FAILURE')
+ build_failpost = self.getJobFromHistory('failpost')
+ self.assertEqual(build_failpost.result, 'POST_FAILURE')
+ build_check_vars = self.getJobFromHistory('check-vars')
+ self.assertEqual(build_check_vars.result, 'SUCCESS')
+ build_hello = self.getJobFromHistory('hello-world')
+ self.assertEqual(build_hello.result, 'SUCCESS')
+ build_python27 = self.getJobFromHistory('python27')
+ self.assertEqual(build_python27.result, 'SUCCESS')
+ flag_path = os.path.join(self.test_root, build_python27.uuid + '.flag')
self.assertTrue(os.path.exists(flag_path))
- copied_path = os.path.join(self.test_root, build.uuid +
+ copied_path = os.path.join(self.test_root, build_python27.uuid +
'.copied')
self.assertTrue(os.path.exists(copied_path))
- failed_path = os.path.join(self.test_root, build.uuid +
+ failed_path = os.path.join(self.test_root, build_python27.uuid +
'.failed')
self.assertFalse(os.path.exists(failed_path))
- pre_flag_path = os.path.join(self.test_root, build.uuid +
+ pre_flag_path = os.path.join(self.test_root, build_python27.uuid +
'.pre.flag')
self.assertTrue(os.path.exists(pre_flag_path))
- post_flag_path = os.path.join(self.test_root, build.uuid +
+ post_flag_path = os.path.join(self.test_root, build_python27.uuid +
'.post.flag')
self.assertTrue(os.path.exists(post_flag_path))
bare_role_flag_path = os.path.join(self.test_root,
- build.uuid + '.bare-role.flag')
+ build_python27.uuid +
+ '.bare-role.flag')
self.assertTrue(os.path.exists(bare_role_flag_path))
secrets_path = os.path.join(self.test_root,
- build.uuid + '.secrets')
+ build_python27.uuid + '.secrets')
with open(secrets_path) as f:
self.assertEqual(f.read(), "test-username test-password")
+ msg = A.messages[0]
+ success = "{} https://success.example.com/zuul-logs/{}"
+ fail = "{} https://failure.example.com/zuul-logs/{}"
+ self.assertIn(success.format("python27", build_python27.uuid), msg)
+ self.assertIn(fail.format("faillocal", build_faillocal.uuid), msg)
+ self.assertIn(success.format("check-vars", build_check_vars.uuid), msg)
+ self.assertIn(success.format("hello-world", build_hello.uuid), msg)
+ self.assertIn(fail.format("timeout", build_timeout.uuid), msg)
+ self.assertIn(fail.format("failpost", build_failpost.uuid), msg)
+
class TestPrePlaybooks(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
old mode 100644
new mode 100755
index e36b24e..72429e9
--- a/tools/encrypt_secret.py
+++ b/tools/encrypt_secret.py
@@ -13,11 +13,19 @@
# under the License.
import argparse
+import base64
import os
import subprocess
import sys
import tempfile
-import urllib
+
+# we to import Request and urlopen differently for python 2 and 3
+try:
+ from urllib.request import Request
+ from urllib.request import urlopen
+except ImportError:
+ from urllib2 import Request
+ from urllib2 import urlopen
DESCRIPTION = """Encrypt a secret for Zuul.
@@ -50,9 +58,9 @@
"to standard output.")
args = parser.parse_args()
- req = urllib.request.Request("%s/keys/%s/%s.pub" % (
+ req = Request("%s/keys/%s/%s.pub" % (
args.url, args.source, args.project))
- pubkey = urllib.request.urlopen(req)
+ pubkey = urlopen(req)
if args.infile:
with open(args.infile) as f:
@@ -70,18 +78,18 @@
pubkey_file.name],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
- (stdout, stderr) = p.communicate(plaintext)
+ (stdout, stderr) = p.communicate(plaintext.encode("utf-8"))
if p.returncode != 0:
raise Exception("Return code %s from openssl" % p.returncode)
- ciphertext = stdout.encode('base64')
+ ciphertext = base64.b64encode(stdout)
finally:
os.unlink(pubkey_file.name)
if args.outfile:
- with open(args.outfile, "w") as f:
+ with open(args.outfile, "wb") as f:
f.write(ciphertext)
else:
- print(ciphertext)
+ print(ciphertext.decode("utf-8"))
if __name__ == '__main__':
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 84227f8..4246206 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1134,7 +1134,7 @@
job = merger.getFiles(
project.source.connection.connection_name,
project.name, branch,
- files=['.zuul.yaml'])
+ files=['zuul.yaml', '.zuul.yaml'])
job.source_context = model.SourceContext(
project, branch, '', False)
jobs.append(job)
@@ -1324,15 +1324,16 @@
def _loadDynamicProjectData(self, config, project, files, trusted):
if trusted:
branches = ['master']
- fn = 'zuul.yaml'
else:
branches = project.source.getProjectBranches(project)
- fn = '.zuul.yaml'
for branch in branches:
incdata = None
- data = files.getFile(project.source.connection.connection_name,
- project.name, branch, fn)
+ for fn in ['zuul.yaml', '.zuul.yaml']:
+ data = files.getFile(project.source.connection.connection_name,
+ project.name, branch, fn)
+ if data:
+ break
if data:
source_context = model.SourceContext(project, branch,
fn, trusted)
diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py
index 95b09e0..5ec2448 100644
--- a/zuul/driver/bubblewrap/__init__.py
+++ b/zuul/driver/bubblewrap/__init__.py
@@ -70,34 +70,11 @@
name = 'bubblewrap'
log = logging.getLogger("zuul.BubblewrapDriver")
- bwrap_command = [
- 'bwrap',
- '--dir', '/tmp',
- '--tmpfs', '/tmp',
- '--dir', '/var',
- '--dir', '/var/tmp',
- '--dir', '/run/user/{uid}',
- '--ro-bind', '/usr', '/usr',
- '--ro-bind', '/lib', '/lib',
- '--ro-bind', '/lib64', '/lib64',
- '--ro-bind', '/bin', '/bin',
- '--ro-bind', '/sbin', '/sbin',
- '--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf',
- '--ro-bind', '{ssh_auth_sock}', '{ssh_auth_sock}',
- '--dir', '{work_dir}',
- '--bind', '{work_dir}', '{work_dir}',
- '--dev', '/dev',
- '--chdir', '{work_dir}',
- '--unshare-all',
- '--share-net',
- '--die-with-parent',
- '--uid', '{uid}',
- '--gid', '{gid}',
- '--file', '{uid_fd}', '/etc/passwd',
- '--file', '{gid_fd}', '/etc/group',
- ]
mounts_map = {'rw': [], 'ro': []}
+ def __init__(self):
+ self.bwrap_command = self._bwrap_command()
+
def reconfigure(self, tenant):
pass
@@ -160,6 +137,38 @@
return wrapped_popen
+ def _bwrap_command(self):
+ bwrap_command = [
+ 'bwrap',
+ '--dir', '/tmp',
+ '--tmpfs', '/tmp',
+ '--dir', '/var',
+ '--dir', '/var/tmp',
+ '--dir', '/run/user/{uid}',
+ '--ro-bind', '/usr', '/usr',
+ '--ro-bind', '/lib', '/lib',
+ '--ro-bind', '/bin', '/bin',
+ '--ro-bind', '/sbin', '/sbin',
+ '--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf',
+ '--ro-bind', '{ssh_auth_sock}', '{ssh_auth_sock}',
+ '--dir', '{work_dir}',
+ '--bind', '{work_dir}', '{work_dir}',
+ '--dev', '/dev',
+ '--chdir', '{work_dir}',
+ '--unshare-all',
+ '--share-net',
+ '--die-with-parent',
+ '--uid', '{uid}',
+ '--gid', '{gid}',
+ '--file', '{uid_fd}', '/etc/passwd',
+ '--file', '{gid_fd}', '/etc/group',
+ ]
+
+ if os.path.isdir('/lib64'):
+ bwrap_command.extend(['--ro-bind', '/lib64', '/lib64'])
+
+ return bwrap_command
+
def main(args=None):
logging.basicConfig(level=logging.DEBUG)
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 838cba5..1a9e37b 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -75,6 +75,8 @@
try:
self.__dispatch_event(request)
+ except webob.exc.HTTPNotFound:
+ raise
except:
self.log.exception("Exception handling Github event:")
@@ -92,7 +94,8 @@
except AttributeError:
message = "Unhandled X-Github-Event: {0}".format(event)
self.log.debug(message)
- raise webob.exc.HTTPBadRequest(message)
+ # Returns empty 200 on unhandled events
+ raise webob.exc.HTTPOk()
try:
json_body = request.json_body
@@ -117,6 +120,8 @@
try:
event = method(json_body)
+ except webob.exc.HTTPNotFound:
+ raise
except:
self.log.exception('Exception when handling event:')
event = None
@@ -219,6 +224,14 @@
event.action = body.get('action')
return event
+ def _event_ping(self, body):
+ project_name = body['repository']['full_name']
+ if not self.connection.getProject(project_name):
+ self.log.warning("Ping received for unknown project %s" %
+ project_name)
+ raise webob.exc.HTTPNotFound("Sorry, this project is not "
+ "registered")
+
def _event_status(self, body):
action = body.get('action')
if action == 'pending':
@@ -340,9 +353,9 @@
self._change_cache = {}
self.projects = {}
self.git_ssh_key = self.connection_config.get('sshkey')
- self.git_host = self.connection_config.get('git_host', 'github.com')
+ self.server = self.connection_config.get('server', 'github.com')
self.canonical_hostname = self.connection_config.get(
- 'canonical_hostname', self.git_host)
+ 'canonical_hostname', self.server)
self.source = driver.getSource(self)
self._github = None
@@ -362,7 +375,7 @@
# The regex is based on the connection host. We do not yet support
# cross-connection dependency gathering
self.depends_on_re = re.compile(
- r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.git_host,
+ r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.server,
re.MULTILINE | re.IGNORECASE)
def onLoad(self):
@@ -375,8 +388,8 @@
self.unregisterHttpHandler(self.payload_path)
def _createGithubClient(self):
- if self.git_host != 'github.com':
- url = 'https://%s/' % self.git_host
+ if self.server != 'github.com':
+ url = 'https://%s/' % self.server
github = github3.GitHubEnterprise(url)
else:
github = github3.GitHub()
@@ -551,7 +564,7 @@
# This leaves off the protocol, but looks for the specific GitHub
# hostname, the org/project, and the pull request number.
- pattern = 'Depends-On %s/%s/pull/%s' % (self.git_host,
+ pattern = 'Depends-On %s/%s/pull/%s' % (self.server,
change.project.name,
change.number)
query = '%s type:pr is:open in:body' % pattern
@@ -595,6 +608,9 @@
change.number)
change.labels = change.pr.get('labels')
change.body = change.pr.get('body')
+ # ensure body is at least an empty string
+ if not change.body:
+ change.body = ''
if history is None:
history = []
@@ -639,18 +655,18 @@
def getGitUrl(self, project):
if self.git_ssh_key:
- return 'ssh://git@%s/%s.git' % (self.git_host, project)
+ return 'ssh://git@%s/%s.git' % (self.server, project)
if self.app_id:
installation_key = self._get_installation_key(project)
return 'https://x-access-token:%s@%s/%s' % (installation_key,
- self.git_host,
+ self.server,
project)
- return 'https://%s/%s' % (self.git_host, project)
+ return 'https://%s/%s' % (self.server, project)
def getGitwebUrl(self, project, sha=None):
- url = 'https://%s/%s' % (self.git_host, project)
+ url = 'https://%s/%s' % (self.server, project)
if sha is not None:
url += '/commit/%s' % sha
return url
@@ -763,8 +779,19 @@
# if there are multiple reviews per user, keep the newest
# note that this breaks the ability to set the 'older-than'
# option on a review requirement.
+ # BUT do not keep the latest if it's a 'commented' type and the
+ # previous review was 'approved' or 'changes_requested', as
+ # the GitHub model does not change the vote if a comment is
+ # added after the fact. THANKS GITHUB!
if review['grantedOn'] > reviews[user]['grantedOn']:
- reviews[user] = review
+ if (review['type'] == 'commented' and reviews[user]['type']
+ in ('approved', 'changes_requested')):
+ self.log.debug("Discarding comment review %s due to "
+ "an existing vote %s" % (review,
+ reviews[user]))
+ pass
+ else:
+ reviews[user] = review
return reviews.values()
@@ -785,7 +812,7 @@
return GithubUser(self.getGithubClient(), login)
def getUserUri(self, login):
- return 'https://%s/%s' % (self.git_host, login)
+ return 'https://%s/%s' % (self.server, login)
def getRepoPermission(self, project, login):
github = self.getGithubClient(project)
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 72087bf..ea41ccd 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -85,8 +85,8 @@
url_pattern = self.config.get('status-url')
if not url_pattern:
sched_config = self.connection.sched.config
- if sched_config.has_option('zuul', 'status_url'):
- url_pattern = sched_config.get('zuul', 'status_url')
+ if sched_config.has_option('webapp', 'status_url'):
+ url_pattern = sched_config.get('webapp', 'status_url')
url = item.formatUrlPattern(url_pattern) if url_pattern else ''
description = ''
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index aaef34e..1c2d202 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -321,9 +321,9 @@
make_project_dict(project,
job_project.override_branch))
projects.add(project)
- for item in all_items:
- if item.change.project not in projects:
- project = item.change.project
+ for i in all_items:
+ if i.change.project not in projects:
+ project = i.change.project
params['projects'].append(make_project_dict(project))
projects.add(project)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index c5d292a..6c390db 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -739,12 +739,11 @@
self.log.exception("Error stopping SSH agent:")
def _execute(self):
- self.log.debug("Job %s: beginning" % (self.job.unique,))
- self.log.debug("Job %s: args: %s" % (self.job.unique,
- self.job.arguments,))
- self.log.debug("Job %s: job root at %s" %
- (self.job.unique, self.jobdir.root))
args = json.loads(self.job.arguments)
+ self.log.debug("Beginning job %s for ref %s" %
+ (self.job.name, args['vars']['zuul']['ref']))
+ self.log.debug("Args: %s" % (self.job.arguments,))
+ self.log.debug("Job root: %s" % (self.jobdir.root,))
tasks = []
projects = set()
diff --git a/zuul/model.py b/zuul/model.py
index dc04e59..436a9c8 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1620,7 +1620,7 @@
result = job.success_message
if job.success_url:
pattern = job.success_url
- elif result == 'FAILURE':
+ else:
if job.failure_message:
result = job.failure_message
if job.failure_url:
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 0ac5766..95b9208 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -14,6 +14,7 @@
import abc
import logging
+from zuul.lib.config import get_default
class BaseReporter(object, metaclass=abc.ABCMeta):
@@ -69,10 +70,8 @@
return ret
def _formatItemReportStart(self, item, with_jobs=True):
- status_url = ''
- if self.connection.sched.config.has_option('zuul', 'status_url'):
- status_url = self.connection.sched.config.get('zuul',
- 'status_url')
+ status_url = get_default(self.connection.sched.config,
+ 'webapp', 'status_url', '')
return item.pipeline.start_message.format(pipeline=item.pipeline,
status_url=status_url)
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a0a93ca..fe6a673 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -547,6 +547,8 @@
else:
items_to_remove.append(item)
for item in items_to_remove:
+ self.log.warning(
+ "Removing item %s during reconfiguration" % (item,))
for build in item.current_build_set.getBuilds():
builds_to_cancel.append(build)
for build in builds_to_cancel: