Merge "Consume tox instead of zuul-tox" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index b91e7e7..a24b833 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -54,18 +54,14 @@
zookeeper
"""""""""
+.. NOTE: this is a white lie at this point, since only the scheduler
+ uses this, however, we expect other components to use it later, so
+ it's reasonable for admins to plan for this now.
+
**hosts**
A list of zookeeper hosts for Zuul to use when communicating with
Nodepool. ``hosts=zk1.example.com,zk2.example.com,zk3.example.com``
-zuul
-""""
-
-**status_url**
- URL that will be posted in Zuul comments made to changes when
- starting jobs for a change. Used by zuul-scheduler only.
- ``status_url=https://zuul.example.com/status``
-
Scheduler
---------
@@ -126,6 +122,11 @@
optional value and ``1`` is used by default.
``status_expiry=1``
+**status_url**
+ URL that will be posted in Zuul comments made to changes when
+ starting jobs for a change. Used by zuul-scheduler only.
+ ``status_url=https://zuul.example.com/status``
+
scheduler
"""""""""
diff --git a/doc/source/admin/drivers/gerrit.rst b/doc/source/admin/drivers/gerrit.rst
index 470b4e8..29e136b 100644
--- a/doc/source/admin/drivers/gerrit.rst
+++ b/doc/source/admin/drivers/gerrit.rst
@@ -35,12 +35,12 @@
**canonical_hostname**
The canonical hostname associated with the git repos on the Gerrit
server. Defaults to the value of **server**. This is used to
- identify repos from this connection by name and in preparing repos
- on the filesystem for use by jobs. This only needs to be set in the
- case where the canonical public location of the git repos is not the
- same as the Gerrit server and it would be incorrect to refer to
- those repos in configuration and build scripts using the Gerrit
- server hostname.
+ identify projects from this connection by name and in preparing
+ repos on the filesystem for use by jobs. Note that Zuul will still
+ only communicate with the Gerrit server identified by **server**;
+ this option is useful if users customarily use a different hostname
+ to clone or pull git repos so that when Zuul places them in the
+ job's working directory, they appear under this directory name.
``canonical_hostname=git.example.com``
**port**
diff --git a/doc/source/admin/drivers/github.rst b/doc/source/admin/drivers/github.rst
index 0cbf895..9740292 100644
--- a/doc/source/admin/drivers/github.rst
+++ b/doc/source/admin/drivers/github.rst
@@ -41,20 +41,20 @@
Path to SSH key to use when cloning github repositories.
``sshkey=/home/zuul/.ssh/id_rsa``
-**git_host**
+**server**
Optional: Hostname of the github install (such as a GitHub Enterprise)
If not specified, defaults to ``github.com``
- ``git_host=github.myenterprise.com``
+ ``server=github.myenterprise.com``
**canonical_hostname**
The canonical hostname associated with the git repos on the GitHub
- server. Defaults to the value of **git_host**. This is used to
- identify repos from this connection by name and in preparing repos
- on the filesystem for use by jobs. This only needs to be set in the
- case where the canonical public location of the git repos is not the
- same as the GitHub server and it would be incorrect to refer to
- those repos in configuration and build scripts using the GitHub
- server hostname.
+ server. Defaults to the value of **server**. This is used to
+ identify projects from this connection by name and in preparing
+ repos on the filesystem for use by jobs. Note that Zuul will still
+ only communicate with the GitHub server identified by **server**;
+ this option is useful if users customarily use a different hostname
+ to clone or pull git repos so that when Zuul places them in the
+ job's working directory, they appear under this directory name.
``canonical_hostname=git.example.com``
Trigger Configuration
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index e7226e9..0b2b5d4 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -99,7 +99,7 @@
+1, or if at least one of them fails, a -1::
- pipeline:
- name: check
+ name: check
manager: independent
trigger:
my_gerrit:
@@ -164,6 +164,17 @@
For more detail on the theory and operation of Zuul's dependent
pipeline manager, see: :doc:`gating`.
+**allow-secrets**
+ This is a boolean which can be used to prevent jobs which require
+ secrets from running in this pipeline. Some pipelines run on
+ proposed changes and therefore execute code which has not yet been
+ reviewed. In such a case, allowing a job to use a secret could
+ result in that secret being exposed. The default is False, meaning
+ that in order to run jobs with secrets, this must be explicitly
+ enabled on each Pipeline where that is safe.
+
+ For more information, see :ref:`secret`.
+
**description**
This field may be used to provide a textual description of the
pipeline. It may appear in the status page or in documentation.
diff --git a/tests/base.py b/tests/base.py
index 617169a..1d55669 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -922,7 +922,7 @@
'http://localhost:%s/connection/%s/payload'
% (port, self.connection_name),
data=payload, headers=headers)
- urllib.request.urlopen(req)
+ return urllib.request.urlopen(req)
def getPull(self, project, number):
pr = self.pull_requests[number - 1]
@@ -1048,7 +1048,7 @@
def _getNeededByFromPR(self, change):
prs = []
pattern = re.compile(r"Depends-On.*https://%s/%s/pull/%s" %
- (self.git_host, change.project.name,
+ (self.server, change.project.name,
change.number))
for pr in self.pull_requests:
if pattern.search(pr.body):
diff --git a/tests/fixtures/zuul-connections-merger.conf b/tests/fixtures/zuul-connections-merger.conf
index 4499493..df465d5 100644
--- a/tests/fixtures/zuul-connections-merger.conf
+++ b/tests/fixtures/zuul-connections-merger.conf
@@ -1,7 +1,7 @@
[gearman]
server=127.0.0.1
-[zuul]
+[webapp]
status_url=http://zuul.example.com/status
[merger]
diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf
index dc28f98..3d61ab6 100644
--- a/tests/fixtures/zuul-github-driver.conf
+++ b/tests/fixtures/zuul-github-driver.conf
@@ -1,7 +1,7 @@
[gearman]
server=127.0.0.1
-[zuul]
+[webapp]
status_url=http://zuul.example.com/status/#{change.number},{change.patchset}
[merger]
@@ -23,4 +23,4 @@
[connection github_ent]
driver=github
sshkey=/home/zuul/.ssh/id_rsa
-git_host=github.enterprise.io
+server=github.enterprise.io
diff --git a/tests/fixtures/zuul-push-reqs.conf b/tests/fixtures/zuul-push-reqs.conf
index c5272aa..4faac13 100644
--- a/tests/fixtures/zuul-push-reqs.conf
+++ b/tests/fixtures/zuul-push-reqs.conf
@@ -1,7 +1,7 @@
[gearman]
server=127.0.0.1
-[zuul]
+[webapp]
status_url=http://zuul.example.com/status
[merger]
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index a19073c..f360866 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -14,6 +14,7 @@
import re
from testtools.matchers import MatchesRegex, StartsWith
+import urllib
import time
from tests.base import ZuulTestCase, simple_layout, random_sha1
@@ -584,3 +585,18 @@
new = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
# New timestamp should be greater than the old timestamp
self.assertLess(old, new)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_ping_event(self):
+ # Test valid ping
+ pevent = {'repository': {'full_name': 'org/project'}}
+ req = self.fake_github.emitEvent(('ping', pevent))
+ self.assertEqual(req.status, 200, "Ping event didn't succeed")
+
+ # Test invalid ping
+ pevent = {'repository': {'full_name': 'unknown-project'}}
+ self.assertRaises(
+ urllib.error.HTTPError,
+ self.fake_github.emitEvent,
+ ('ping', pevent),
+ )
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 84227f8..4246206 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1134,7 +1134,7 @@
job = merger.getFiles(
project.source.connection.connection_name,
project.name, branch,
- files=['.zuul.yaml'])
+ files=['zuul.yaml', '.zuul.yaml'])
job.source_context = model.SourceContext(
project, branch, '', False)
jobs.append(job)
@@ -1324,15 +1324,16 @@
def _loadDynamicProjectData(self, config, project, files, trusted):
if trusted:
branches = ['master']
- fn = 'zuul.yaml'
else:
branches = project.source.getProjectBranches(project)
- fn = '.zuul.yaml'
for branch in branches:
incdata = None
- data = files.getFile(project.source.connection.connection_name,
- project.name, branch, fn)
+ for fn in ['zuul.yaml', '.zuul.yaml']:
+ data = files.getFile(project.source.connection.connection_name,
+ project.name, branch, fn)
+ if data:
+ break
if data:
source_context = model.SourceContext(project, branch,
fn, trusted)
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 838cba5..e35d860 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -75,6 +75,8 @@
try:
self.__dispatch_event(request)
+ except webob.exc.HTTPNotFound:
+ raise
except:
self.log.exception("Exception handling Github event:")
@@ -92,7 +94,8 @@
except AttributeError:
message = "Unhandled X-Github-Event: {0}".format(event)
self.log.debug(message)
- raise webob.exc.HTTPBadRequest(message)
+ # Returns empty 200 on unhandled events
+ raise webob.exc.HTTPOk()
try:
json_body = request.json_body
@@ -117,6 +120,8 @@
try:
event = method(json_body)
+ except webob.exc.HTTPNotFound:
+ raise
except:
self.log.exception('Exception when handling event:')
event = None
@@ -219,6 +224,14 @@
event.action = body.get('action')
return event
+ def _event_ping(self, body):
+ project_name = body['repository']['full_name']
+ if not self.connection.getProject(project_name):
+ self.log.warning("Ping received for unknown project %s" %
+ project_name)
+ raise webob.exc.HTTPNotFound("Sorry, this project is not "
+ "registered")
+
def _event_status(self, body):
action = body.get('action')
if action == 'pending':
@@ -340,9 +353,9 @@
self._change_cache = {}
self.projects = {}
self.git_ssh_key = self.connection_config.get('sshkey')
- self.git_host = self.connection_config.get('git_host', 'github.com')
+ self.server = self.connection_config.get('server', 'github.com')
self.canonical_hostname = self.connection_config.get(
- 'canonical_hostname', self.git_host)
+ 'canonical_hostname', self.server)
self.source = driver.getSource(self)
self._github = None
@@ -362,7 +375,7 @@
# The regex is based on the connection host. We do not yet support
# cross-connection dependency gathering
self.depends_on_re = re.compile(
- r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.git_host,
+ r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.server,
re.MULTILINE | re.IGNORECASE)
def onLoad(self):
@@ -375,8 +388,8 @@
self.unregisterHttpHandler(self.payload_path)
def _createGithubClient(self):
- if self.git_host != 'github.com':
- url = 'https://%s/' % self.git_host
+ if self.server != 'github.com':
+ url = 'https://%s/' % self.server
github = github3.GitHubEnterprise(url)
else:
github = github3.GitHub()
@@ -551,7 +564,7 @@
# This leaves off the protocol, but looks for the specific GitHub
# hostname, the org/project, and the pull request number.
- pattern = 'Depends-On %s/%s/pull/%s' % (self.git_host,
+ pattern = 'Depends-On %s/%s/pull/%s' % (self.server,
change.project.name,
change.number)
query = '%s type:pr is:open in:body' % pattern
@@ -639,18 +652,18 @@
def getGitUrl(self, project):
if self.git_ssh_key:
- return 'ssh://git@%s/%s.git' % (self.git_host, project)
+ return 'ssh://git@%s/%s.git' % (self.server, project)
if self.app_id:
installation_key = self._get_installation_key(project)
return 'https://x-access-token:%s@%s/%s' % (installation_key,
- self.git_host,
+ self.server,
project)
- return 'https://%s/%s' % (self.git_host, project)
+ return 'https://%s/%s' % (self.server, project)
def getGitwebUrl(self, project, sha=None):
- url = 'https://%s/%s' % (self.git_host, project)
+ url = 'https://%s/%s' % (self.server, project)
if sha is not None:
url += '/commit/%s' % sha
return url
@@ -785,7 +798,7 @@
return GithubUser(self.getGithubClient(), login)
def getUserUri(self, login):
- return 'https://%s/%s' % (self.git_host, login)
+ return 'https://%s/%s' % (self.server, login)
def getRepoPermission(self, project, login):
github = self.getGithubClient(project)
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 72087bf..ea41ccd 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -85,8 +85,8 @@
url_pattern = self.config.get('status-url')
if not url_pattern:
sched_config = self.connection.sched.config
- if sched_config.has_option('zuul', 'status_url'):
- url_pattern = sched_config.get('zuul', 'status_url')
+ if sched_config.has_option('webapp', 'status_url'):
+ url_pattern = sched_config.get('webapp', 'status_url')
url = item.formatUrlPattern(url_pattern) if url_pattern else ''
description = ''
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index aaef34e..1c2d202 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -321,9 +321,9 @@
make_project_dict(project,
job_project.override_branch))
projects.add(project)
- for item in all_items:
- if item.change.project not in projects:
- project = item.change.project
+ for i in all_items:
+ if i.change.project not in projects:
+ project = i.change.project
params['projects'].append(make_project_dict(project))
projects.add(project)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index c5d292a..6c390db 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -739,12 +739,11 @@
self.log.exception("Error stopping SSH agent:")
def _execute(self):
- self.log.debug("Job %s: beginning" % (self.job.unique,))
- self.log.debug("Job %s: args: %s" % (self.job.unique,
- self.job.arguments,))
- self.log.debug("Job %s: job root at %s" %
- (self.job.unique, self.jobdir.root))
args = json.loads(self.job.arguments)
+ self.log.debug("Beginning job %s for ref %s" %
+ (self.job.name, args['vars']['zuul']['ref']))
+ self.log.debug("Args: %s" % (self.job.arguments,))
+ self.log.debug("Job root: %s" % (self.jobdir.root,))
tasks = []
projects = set()
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 0ac5766..95b9208 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -14,6 +14,7 @@
import abc
import logging
+from zuul.lib.config import get_default
class BaseReporter(object, metaclass=abc.ABCMeta):
@@ -69,10 +70,8 @@
return ret
def _formatItemReportStart(self, item, with_jobs=True):
- status_url = ''
- if self.connection.sched.config.has_option('zuul', 'status_url'):
- status_url = self.connection.sched.config.get('zuul',
- 'status_url')
+ status_url = get_default(self.connection.sched.config,
+ 'webapp', 'status_url', '')
return item.pipeline.start_message.format(pipeline=item.pipeline,
status_url=status_url)