Merge "Remove uncalled test code" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index 0ae5beb..c21b30f 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -6,4 +6,5 @@
- tox-cover
- tox-linters
- tox-py27
+ - tox-py35
- tox-tarball
diff --git a/doc/source/connections.rst b/doc/source/connections.rst
index 614b44a..120d529 100644
--- a/doc/source/connections.rst
+++ b/doc/source/connections.rst
@@ -65,6 +65,33 @@
be added to Gerrit. Zuul is very flexible and can take advantage of
those.
+GitHub
+------
+
+Create a connection with GitHub.
+
+**driver=github**
+
+**api_token**
+ API token for accessing GitHub.
+ See `Creating an access token for command-line use
+ <https://help.github.com/articles/creating-an-access-token-for-command-line-use/>`_.
+
+**webhook_token**
+ Optional: Token for validating the webhook event payloads.
+ If not specified, payloads are not validated.
+ See `Securing your webhooks
+ <https://developer.github.com/webhooks/securing/>`_.
+
+**sshkey**
+ Path to SSH key to use when cloning github repositories.
+ ``sshkey=/home/zuul/.ssh/id_rsa``
+
+**git_host**
+ Optional: Hostname of the github install (such as a GitHub Enterprise)
+ If not specified, defaults to ``github.com``
+ ``git_host=github.myenterprise.com``
+
SMTP
----
diff --git a/doc/source/reporters.rst b/doc/source/reporters.rst
index b01c8d1..e3ab947 100644
--- a/doc/source/reporters.rst
+++ b/doc/source/reporters.rst
@@ -28,6 +28,43 @@
A :ref:`connection` that uses the gerrit driver must be supplied to the
trigger.
+GitHub
+------
+
+Zuul reports back to GitHub pull requests via GitHub API.
+On success and failure, it creates a comment containing the build results.
+It also sets the status on start, success and failure. Status name and
+description is taken from the pipeline.
+
+A :ref:`connection` that uses the github driver must be supplied to the
+reporter. It has the following options:
+
+ **status**
+ String value (``pending``, ``success``, ``failure``) that the reporter should
+ set as the commit status on github.
+ ``status: 'success'``
+
+ **comment**
+ Boolean value (``true`` or ``false``) that determines if the reporter should
+ add a comment to the pipeline status to the github pull request. Defaults
+ to ``true``.
+ ``comment: false``
+
+ **merge**
+ Boolean value (``true`` or ``false``) that determines if the reporter should
+ merge the pull reqeust. Defaults to ``false``.
+ ``merge=true``
+
+ **label**
+ List of strings each representing an exact label name which should be added
+ to the pull request by reporter.
+ ``label: 'test successful'``
+
+ **unlabel**
+ List of strings each representing an exact label name which should be removed
+ from the pull request by reporter.
+ ``unlabel: 'test failed'``
+
SMTP
----
diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst
index 263f280..f73ad2f 100644
--- a/doc/source/triggers.rst
+++ b/doc/source/triggers.rst
@@ -4,7 +4,7 @@
========
The process of merging a change starts with proposing a change to be
-merged. Primarily, Zuul supports Gerrit as a triggering system.
+merged. Zuul supports Gerrit and GitHub as triggering systems.
Zuul's design is modular, so alternate triggering and reporting
systems can be supported.
@@ -100,6 +100,88 @@
*require-approval* but will fail to enter the pipeline if there is
a matching approval.
+GitHub
+------
+
+Github webhook events can be configured as triggers.
+
+A connection name with the github driver can take multiple events with the
+following options.
+
+ **event**
+ The event from github. Supported events are ``pull_request``,
+ ``pull_request_review``, and ``push``.
+
+ A ``pull_request`` event will
+ have associated action(s) to trigger from. The supported actions are:
+
+ *opened* - pull request opened
+
+ *changed* - pull request synchronized
+
+ *closed* - pull request closed
+
+ *reopened* - pull request reopened
+
+ *comment* - comment added on pull request
+
+ *labeled* - label added on pull request
+
+ *unlabeled* - label removed from pull request
+
+ *review* - review added on pull request
+
+ *push* - head reference updated (pushed to branch)
+
+ A ``pull_request_review`` event will
+ have associated action(s) to trigger from. The supported actions are:
+
+ *submitted* - pull request review added
+
+ *dismissed* - pull request review removed
+
+ **branch**
+ The branch associated with the event. Example: ``master``. This
+ field is treated as a regular expression, and multiple branches may
+ be listed. Used for ``pull_request`` and ``pull_request_review`` events.
+
+ **comment**
+ This is only used for ``pull_request`` ``comment`` actions. It accepts a
+ list of regexes that are searched for in the comment string. If any of these
+ regexes matches a portion of the comment string the trigger is matched.
+ ``comment: retrigger`` will match when comments containing 'retrigger'
+ somewhere in the comment text are added to a pull request.
+
+ **label**
+ This is only used for ``labeled`` and ``unlabeled`` ``pull_request`` actions.
+ It accepts a list of strings each of which matches the label name in the
+ event literally. ``label: recheck`` will match a ``labeled`` action when
+ pull request is labeled with a ``recheck`` label. ``label: 'do not test'``
+ will match a ``unlabeled`` action when a label with name ``do not test`` is
+ removed from the pull request.
+
+ **state**
+ This is only used for ``pull_request_review`` events. It accepts a list of
+ strings each of which is matched to the review state, which can be one of
+ ``approved``, ``comment``, or ``request_changes``.
+
+ **ref**
+ This is only used for ``push`` events. This field is treated as a regular
+ expression and multiple refs may be listed. Github always sends full ref
+ name, eg. ``refs/tags/bar`` and this string is matched against the regexp.
+
+GitHub Configuration
+~~~~~~~~~~~~~~~~~~~~
+
+Configure GitHub `webhook events
+<https://developer.github.com/webhooks/creating/>`_.
+
+Set *Payload URL* to
+``http://<zuul-hostname>/connection/<connection-name>/payload``.
+
+Set *Content Type* to ``application/json``.
+
+Select *Events* you are interested in. See above for the supported events.
Timer
-----
@@ -154,4 +236,4 @@
*reject-approval*
This takes a list of approvals in the same format as
*require-approval* but will fail to enter the pipeline if there is
- a matching approval.
\ No newline at end of file
+ a matching approval.
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index d973948..5c69bd1 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -52,6 +52,10 @@
var collapsed_exceptions = [];
var current_filter = read_cookie('zuul_filter_string', '');
+ var change_set_in_url = window.location.href.split('#')[1];
+ if (change_set_in_url) {
+ current_filter = change_set_in_url;
+ }
var $jq;
var xhr,
diff --git a/requirements.txt b/requirements.txt
index 974b77f..9f20458 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
pbr>=1.1.0
+Github3.py==1.0.0a2
PyYAML>=3.1.0
Paste
WebOb>=1.2.3
@@ -10,7 +11,7 @@
extras
statsd>=1.0.0,<3.0
voluptuous>=0.10.2
-gear>=0.5.7,<1.0.0
+gear>=0.9.0,<1.0.0
apscheduler>=3.0
PrettyTable>=0.6,<0.8
babel>=1.0
diff --git a/test-requirements.txt b/test-requirements.txt
index 6262a02..baf6cad 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,7 +1,9 @@
-hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0
+pep8
+pyflakes
+flake8
coverage>=3.6
-sphinx>=1.5.1
+sphinx>=1.5.1,<1.6
sphinxcontrib-blockdiag>=1.1.0
fixtures>=0.3.14
python-keystoneclient>=0.4.2
diff --git a/tests/base.py b/tests/base.py
index f8cfd02..0105ffa 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -58,6 +58,7 @@
import zuul.driver.gerrit.gerritsource as gerritsource
import zuul.driver.gerrit.gerritconnection as gerritconnection
+import zuul.driver.github.githubconnection as githubconnection
import zuul.scheduler
import zuul.webapp
import zuul.rpclistener
@@ -69,6 +70,7 @@
import zuul.merger.server
import zuul.nodepool
import zuul.zk
+from zuul.exceptions import MergeFailure
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'fixtures')
@@ -88,7 +90,7 @@
def random_sha1():
- return hashlib.sha1(str(random.random())).hexdigest()
+ return hashlib.sha1(str(random.random()).encode('ascii')).hexdigest()
def iterate_timeout(max_seconds, purpose):
@@ -126,12 +128,12 @@
return decorator
-class ChangeReference(git.Reference):
+class GerritChangeReference(git.Reference):
_common_path_default = "refs/changes"
_points_to_commits_only = True
-class FakeChange(object):
+class FakeGerritChange(object):
categories = {'approved': ('Approved', -1, 1),
'code-review': ('Code-Review', -2, 2),
'verified': ('Verified', -2, 2)}
@@ -139,6 +141,7 @@
def __init__(self, gerrit, number, project, branch, subject,
status='NEW', upstream_root=None, files={}):
self.gerrit = gerrit
+ self.source = gerrit
self.reported = 0
self.queried = 0
self.patchsets = []
@@ -178,9 +181,9 @@
def addFakeChangeToRepo(self, msg, files, large):
path = os.path.join(self.upstream_root, self.project)
repo = git.Repo(path)
- ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
- self.latest_patchset),
- 'refs/tags/init')
+ ref = GerritChangeReference.create(
+ repo, '1/%s/%s' % (self.number, self.latest_patchset),
+ 'refs/tags/init')
repo.head.reference = ref
zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
@@ -469,9 +472,9 @@
files=None):
"""Add a change to the fake Gerrit."""
self.change_number += 1
- c = FakeChange(self, self.change_number, project, branch, subject,
- upstream_root=self.upstream_root,
- status=status, files=files)
+ c = FakeGerritChange(self, self.change_number, project, branch,
+ subject, upstream_root=self.upstream_root,
+ status=status, files=files)
self.changes[self.change_number] = c
return c
@@ -531,6 +534,391 @@
return os.path.join(self.upstream_root, project.name)
+class GithubChangeReference(git.Reference):
+ _common_path_default = "refs/pull"
+ _points_to_commits_only = True
+
+
+class FakeGithubPullRequest(object):
+
+ def __init__(self, github, number, project, branch,
+ subject, upstream_root, files=[], number_of_commits=1):
+ """Creates a new PR with several commits.
+ Sends an event about opened PR."""
+ self.github = github
+ self.source = github
+ self.number = number
+ self.project = project
+ self.branch = branch
+ self.subject = subject
+ self.number_of_commits = 0
+ self.upstream_root = upstream_root
+ self.files = []
+ self.comments = []
+ self.labels = []
+ self.statuses = {}
+ self.updated_at = None
+ self.head_sha = None
+ self.is_merged = False
+ self.merge_message = None
+ self._createPRRef()
+ self._addCommitToRepo(files=files)
+ self._updateTimeStamp()
+
+ def addCommit(self, files=[]):
+ """Adds a commit on top of the actual PR head."""
+ self._addCommitToRepo(files=files)
+ self._updateTimeStamp()
+ self._clearStatuses()
+
+ def forcePush(self, files=[]):
+ """Clears actual commits and add a commit on top of the base."""
+ self._addCommitToRepo(files=files, reset=True)
+ self._updateTimeStamp()
+ self._clearStatuses()
+
+ def getPullRequestOpenedEvent(self):
+ return self._getPullRequestEvent('opened')
+
+ def getPullRequestSynchronizeEvent(self):
+ return self._getPullRequestEvent('synchronize')
+
+ def getPullRequestReopenedEvent(self):
+ return self._getPullRequestEvent('reopened')
+
+ def getPullRequestClosedEvent(self):
+ return self._getPullRequestEvent('closed')
+
+ def addComment(self, message):
+ self.comments.append(message)
+ self._updateTimeStamp()
+
+ def getCommentAddedEvent(self, text):
+ name = 'issue_comment'
+ data = {
+ 'action': 'created',
+ 'issue': {
+ 'number': self.number
+ },
+ 'comment': {
+ 'body': text
+ },
+ 'repository': {
+ 'full_name': self.project
+ },
+ 'sender': {
+ 'login': 'ghuser'
+ }
+ }
+ return (name, data)
+
+ def getReviewAddedEvent(self, review):
+ name = 'pull_request_review'
+ data = {
+ 'action': 'submitted',
+ 'pull_request': {
+ 'number': self.number,
+ 'title': self.subject,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ },
+ 'review': {
+ 'state': review
+ },
+ 'repository': {
+ 'full_name': self.project
+ },
+ 'sender': {
+ 'login': 'ghuser'
+ }
+ }
+ return (name, data)
+
+ def addLabel(self, name):
+ if name not in self.labels:
+ self.labels.append(name)
+ self._updateTimeStamp()
+ return self._getLabelEvent(name)
+
+ def removeLabel(self, name):
+ if name in self.labels:
+ self.labels.remove(name)
+ self._updateTimeStamp()
+ return self._getUnlabelEvent(name)
+
+ def _getLabelEvent(self, label):
+ name = 'pull_request'
+ data = {
+ 'action': 'labeled',
+ 'pull_request': {
+ 'number': self.number,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ },
+ 'label': {
+ 'name': label
+ },
+ 'sender': {
+ 'login': 'ghuser'
+ }
+ }
+ return (name, data)
+
+ def _getUnlabelEvent(self, label):
+ name = 'pull_request'
+ data = {
+ 'action': 'unlabeled',
+ 'pull_request': {
+ 'number': self.number,
+ 'title': self.subject,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ },
+ 'label': {
+ 'name': label
+ },
+ 'sender': {
+ 'login': 'ghuser'
+ }
+ }
+ return (name, data)
+
+ def _getRepo(self):
+ repo_path = os.path.join(self.upstream_root, self.project)
+ return git.Repo(repo_path)
+
+ def _createPRRef(self):
+ repo = self._getRepo()
+ GithubChangeReference.create(
+ repo, self._getPRReference(), 'refs/tags/init')
+
+ def _addCommitToRepo(self, files=[], reset=False):
+ repo = self._getRepo()
+ ref = repo.references[self._getPRReference()]
+ if reset:
+ self.number_of_commits = 0
+ ref.set_object('refs/tags/init')
+ self.number_of_commits += 1
+ repo.head.reference = ref
+ zuul.merger.merger.reset_repo_to_head(repo)
+ repo.git.clean('-x', '-f', '-d')
+
+ if files:
+ fn = files[0]
+ self.files = files
+ else:
+ fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
+ self.files = [fn]
+ msg = self.subject + '-' + str(self.number_of_commits)
+ fn = os.path.join(repo.working_dir, fn)
+ f = open(fn, 'w')
+ with open(fn, 'w') as f:
+ f.write("test %s %s\n" %
+ (self.branch, self.number))
+ repo.index.add([fn])
+
+ self.head_sha = repo.index.commit(msg).hexsha
+ repo.head.reference = 'master'
+ zuul.merger.merger.reset_repo_to_head(repo)
+ repo.git.clean('-x', '-f', '-d')
+ repo.heads['master'].checkout()
+
+ def _updateTimeStamp(self):
+ self.updated_at = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
+
+ def getPRHeadSha(self):
+ repo = self._getRepo()
+ return repo.references[self._getPRReference()].commit.hexsha
+
+ def setStatus(self, state, url, description, context):
+ self.statuses[context] = {
+ 'state': state,
+ 'url': url,
+ 'description': description
+ }
+
+ def _clearStatuses(self):
+ self.statuses = {}
+
+ def _getPRReference(self):
+ return '%s/head' % self.number
+
+ def _getPullRequestEvent(self, action):
+ name = 'pull_request'
+ data = {
+ 'action': action,
+ 'number': self.number,
+ 'pull_request': {
+ 'number': self.number,
+ 'title': self.subject,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ },
+ 'sender': {
+ 'login': 'ghuser'
+ }
+ }
+ return (name, data)
+
+
+class FakeGithubConnection(githubconnection.GithubConnection):
+ log = logging.getLogger("zuul.test.FakeGithubConnection")
+
+ def __init__(self, driver, connection_name, connection_config,
+ upstream_root=None):
+ super(FakeGithubConnection, self).__init__(driver, connection_name,
+ connection_config)
+ self.connection_name = connection_name
+ self.pr_number = 0
+ self.pull_requests = []
+ self.upstream_root = upstream_root
+ self.merge_failure = False
+ self.merge_not_allowed_count = 0
+
+ def openFakePullRequest(self, project, branch, subject, files=[]):
+ self.pr_number += 1
+ pull_request = FakeGithubPullRequest(
+ self, self.pr_number, project, branch, subject, self.upstream_root,
+ files=files)
+ self.pull_requests.append(pull_request)
+ return pull_request
+
+ def getPushEvent(self, project, ref, old_rev=None, new_rev=None):
+ if not old_rev:
+ old_rev = '00000000000000000000000000000000'
+ if not new_rev:
+ new_rev = random_sha1()
+ name = 'push'
+ data = {
+ 'ref': ref,
+ 'before': old_rev,
+ 'after': new_rev,
+ 'repository': {
+ 'full_name': project
+ }
+ }
+ return (name, data)
+
+ def emitEvent(self, event):
+ """Emulates sending the GitHub webhook event to the connection."""
+ port = self.webapp.server.socket.getsockname()[1]
+ name, data = event
+ payload = json.dumps(data).encode('utf8')
+ headers = {'X-Github-Event': name}
+ req = urllib.request.Request(
+ 'http://localhost:%s/connection/%s/payload'
+ % (port, self.connection_name),
+ data=payload, headers=headers)
+ urllib.request.urlopen(req)
+
+ def getPull(self, project, number):
+ pr = self.pull_requests[number - 1]
+ data = {
+ 'number': number,
+ 'title': pr.subject,
+ 'updated_at': pr.updated_at,
+ 'base': {
+ 'repo': {
+ 'full_name': pr.project
+ },
+ 'ref': pr.branch,
+ },
+ 'mergeable': True,
+ 'head': {
+ 'sha': pr.head_sha
+ }
+ }
+ return data
+
+ def getPullFileNames(self, project, number):
+ pr = self.pull_requests[number - 1]
+ return pr.files
+
+ def getUser(self, login):
+ data = {
+ 'username': login,
+ 'name': 'Github User',
+ 'email': 'github.user@example.com'
+ }
+ return data
+
+ def getGitUrl(self, project):
+ return os.path.join(self.upstream_root, str(project))
+
+ def real_getGitUrl(self, project):
+ return super(FakeGithubConnection, self).getGitUrl(project)
+
+ def getProjectBranches(self, project):
+ """Masks getProjectBranches since we don't have a real github"""
+
+ # just returns master for now
+ return ['master']
+
+ def commentPull(self, project, pr_number, message):
+ pull_request = self.pull_requests[pr_number - 1]
+ pull_request.addComment(message)
+
+ def mergePull(self, project, pr_number, commit_message='', sha=None):
+ pull_request = self.pull_requests[pr_number - 1]
+ if self.merge_failure:
+ raise Exception('Pull request was not merged')
+ if self.merge_not_allowed_count > 0:
+ self.merge_not_allowed_count -= 1
+ raise MergeFailure('Merge was not successful due to mergeability'
+ ' conflict')
+ pull_request.is_merged = True
+ pull_request.merge_message = commit_message
+
+ def setCommitStatus(self, project, sha, state,
+ url='', description='', context=''):
+ owner, proj = project.split('/')
+ for pr in self.pull_requests:
+ pr_owner, pr_project = pr.project.split('/')
+ if (pr_owner == owner and pr_project == proj and
+ pr.head_sha == sha):
+ pr.setStatus(state, url, description, context)
+
+ def labelPull(self, project, pr_number, label):
+ pull_request = self.pull_requests[pr_number - 1]
+ pull_request.addLabel(label)
+
+ def unlabelPull(self, project, pr_number, label):
+ pull_request = self.pull_requests[pr_number - 1]
+ pull_request.removeLabel(label)
+
+
class BuildHistory(object):
def __init__(self, **kw):
self.__dict__.update(kw)
@@ -588,7 +976,7 @@
return
def stop(self):
- os.write(self.wake_write, '1\n')
+ os.write(self.wake_write, b'1\n')
class FakeBuild(object):
@@ -696,7 +1084,7 @@
"""
for change in changes:
- hostname = change.gerrit.canonical_hostname
+ hostname = change.source.canonical_hostname
path = os.path.join(self.jobdir.src_root, hostname, change.project)
try:
repo = git.Repo(path)
@@ -872,7 +1260,7 @@
for queue in [self.high_queue, self.normal_queue, self.low_queue]:
for job in queue:
if not hasattr(job, 'waiting'):
- if job.name.startswith('executor:execute'):
+ if job.name.startswith(b'executor:execute'):
job.waiting = self.hold_jobs_in_queue
else:
job.waiting = False
@@ -971,7 +1359,10 @@
def run(self):
while self._running:
- self._run()
+ try:
+ self._run()
+ except Exception:
+ self.log.exception("Error in fake nodepool:")
time.sleep(0.1)
def _run(self):
@@ -990,7 +1381,7 @@
path = self.REQUEST_ROOT + '/' + oid
try:
data, stat = self.client.get(path)
- data = json.loads(data)
+ data = json.loads(data.decode('utf8'))
data['_oid'] = oid
reqs.append(data)
except kazoo.exceptions.NoNodeError:
@@ -1006,7 +1397,7 @@
for oid in sorted(nodeids):
path = self.NODE_ROOT + '/' + oid
data, stat = self.client.get(path)
- data = json.loads(data)
+ data = json.loads(data.decode('utf8'))
data['_oid'] = oid
try:
lockfiles = self.client.get_children(path + '/lock')
@@ -1038,7 +1429,7 @@
image_id=None,
host_keys=["fake-key1", "fake-key2"],
executor='fake-nodepool')
- data = json.dumps(data)
+ data = json.dumps(data).encode('utf8')
path = self.client.create(path, data,
makepath=True,
sequence=True)
@@ -1067,9 +1458,12 @@
request['state_time'] = time.time()
path = self.REQUEST_ROOT + '/' + oid
- data = json.dumps(request)
+ data = json.dumps(request).encode('utf8')
self.log.debug("Fulfilling node request: %s %s" % (oid, data))
- self.client.set(path, data)
+ try:
+ self.client.set(path, data)
+ except kazoo.exceptions.NoNodeError:
+ self.log.debug("Node request %s %s disappeared" % (oid, data))
class ChrootedKazooFixture(fixtures.Fixture):
@@ -1228,7 +1622,7 @@
# from libraries that zuul depends on such as gear.
log_defaults_from_env = os.environ.get(
'OS_LOG_DEFAULTS',
- 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
+ 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO,paste=INFO')
if log_defaults_from_env:
for default in log_defaults_from_env.split(','):
@@ -1446,6 +1840,16 @@
'zuul.driver.gerrit.GerritDriver.getConnection',
getGerritConnection))
+ def getGithubConnection(driver, name, config):
+ con = FakeGithubConnection(driver, name, config,
+ upstream_root=self.upstream_root)
+ setattr(self, 'fake_' + name, con)
+ return con
+
+ self.useFixture(fixtures.MonkeyPatch(
+ 'zuul.driver.github.GithubDriver.getConnection',
+ getGithubConnection))
+
# Set up smtp related fakes
# TODO(jhesketh): This should come from lib.connections for better
# coverage
@@ -1525,7 +1929,7 @@
'source': {driver:
{'config-projects': ['common-config'],
'untrusted-projects': untrusted_projects}}}}]
- f.write(yaml.dump(config))
+ f.write(yaml.dump(config).encode('utf8'))
f.close()
self.config.set('zuul', 'tenant_config',
os.path.join(FIXTURE_DIR, f.name))
@@ -1799,8 +2203,9 @@
if build.url is None:
self.log.debug("%s has not reported start" % build)
return False
+ # using internal ServerJob which offers no Text interface
worker_build = self.executor_server.job_builds.get(
- server_job.unique)
+ server_job.unique.decode('utf8'))
if worker_build:
if worker_build.isWaiting():
continue
@@ -1879,7 +2284,7 @@
def countJobResults(self, jobs, result):
jobs = filter(lambda x: x.result == result, jobs)
- return len(jobs)
+ return len(list(jobs))
def getJobFromHistory(self, name, project=None):
for job in self.history:
@@ -1904,7 +2309,7 @@
start = time.time()
while time.time() < (start + 5):
for stat in self.statsd.stats:
- k, v = stat.split(':')
+ k, v = stat.decode('utf-8').split(':')
if key == k:
if value is None and kind is None:
return
diff --git a/tests/fixtures/config/multi-driver/git/common-config/playbooks/project-gerrit.yaml b/tests/fixtures/config/multi-driver/git/common-config/playbooks/project-gerrit.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-driver/git/common-config/playbooks/project-gerrit.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-driver/git/common-config/playbooks/project1-github.yaml b/tests/fixtures/config/multi-driver/git/common-config/playbooks/project1-github.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-driver/git/common-config/playbooks/project1-github.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-driver/git/common-config/zuul.yaml b/tests/fixtures/config/multi-driver/git/common-config/zuul.yaml
new file mode 100644
index 0000000..2dab845
--- /dev/null
+++ b/tests/fixtures/config/multi-driver/git/common-config/zuul.yaml
@@ -0,0 +1,46 @@
+- pipeline:
+ name: check_github
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action:
+ - opened
+ - changed
+ - reopened
+ success:
+ github:
+ status: 'success'
+ failure:
+ github:
+ status: 'failure'
+
+- pipeline:
+ name: check_gerrit
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verify: 1
+ failure:
+ gerrit:
+ verify: 1
+
+- job:
+ name: project-gerrit
+- job:
+ name: project1-github
+
+- project:
+ name: org/project
+ check_gerrit:
+ jobs:
+ - project-gerrit
+
+- project:
+ name: org/project1
+ check_github:
+ jobs:
+ - project1-github
diff --git a/tests/fixtures/config/multi-driver/git/org_project/README b/tests/fixtures/config/multi-driver/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/multi-driver/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/multi-driver/git/org_project1/README b/tests/fixtures/config/multi-driver/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/multi-driver/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/multi-driver/main.yaml b/tests/fixtures/config/multi-driver/main.yaml
new file mode 100644
index 0000000..301df38
--- /dev/null
+++ b/tests/fixtures/config/multi-driver/main.yaml
@@ -0,0 +1,11 @@
+- tenant:
+ name: tenant-one
+ source:
+ github:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ gerrit:
+ untrusted-projects:
+ - org/project
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 34bd9cd..2bb61ee 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -69,7 +69,6 @@
- job:
name: project1-project2-integration
- queue-name: integration
- job:
name: project-testfile
diff --git a/tests/fixtures/layout-live-reconfiguration-add-job.yaml b/tests/fixtures/layout-live-reconfiguration-add-job.yaml
deleted file mode 100644
index e4aea6f..0000000
--- a/tests/fixtures/layout-live-reconfiguration-add-job.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-pipelines:
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- start:
- gerrit:
- verified: 0
- precedence: high
-
-jobs:
- - name: ^.*-merge$
- failure-message: Unable to merge change
- hold-following-changes: true
- - name: project-testfile
- files:
- - '.*-requires'
-
-projects:
- - name: org/project
- merge-mode: cherry-pick
- gate:
- - project-merge:
- - project-test1
- - project-test2
- - project-test3
- - project-testfile
diff --git a/tests/fixtures/layout-live-reconfiguration-failed-job.yaml b/tests/fixtures/layout-live-reconfiguration-failed-job.yaml
deleted file mode 100644
index e811af1..0000000
--- a/tests/fixtures/layout-live-reconfiguration-failed-job.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
-jobs:
- - name: ^.*-merge$
- failure-message: Unable to merge change
- hold-following-changes: true
-
-projects:
- - name: org/project
- merge-mode: cherry-pick
- check:
- - project-merge:
- - project-test2
- - project-testfile
diff --git a/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml b/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml
deleted file mode 100644
index ad3f666..0000000
--- a/tests/fixtures/layout-live-reconfiguration-shared-queue.yaml
+++ /dev/null
@@ -1,62 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- start:
- gerrit:
- verified: 0
- precedence: high
-
-jobs:
- - name: ^.*-merge$
- failure-message: Unable to merge change
- hold-following-changes: true
- - name: project1-project2-integration
- queue-name: integration
-
-projects:
- - name: org/project1
- check:
- - project1-merge:
- - project1-test1
- - project1-test2
- gate:
- - project1-merge:
- - project1-test1
- - project1-test2
-
- - name: org/project2
- check:
- - project2-merge:
- - project2-test1
- - project2-test2
- - project1-project2-integration
- gate:
- - project2-merge:
- - project2-test1
- - project2-test2
- - project1-project2-integration
diff --git a/tests/fixtures/layouts/basic-github.yaml b/tests/fixtures/layouts/basic-github.yaml
new file mode 100644
index 0000000..709fd02
--- /dev/null
+++ b/tests/fixtures/layouts/basic-github.yaml
@@ -0,0 +1,30 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action:
+ - opened
+ - changed
+ - reopened
+ branch: '^master$'
+ - event: pull_request
+ action: comment
+ comment: 'test me'
+ success:
+ github: {}
+ failure:
+ github: {}
+
+- job:
+ name: project-test1
+- job:
+ name: project-test2
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ - project-test2
diff --git a/tests/fixtures/layouts/dependent-github.yaml b/tests/fixtures/layouts/dependent-github.yaml
new file mode 100644
index 0000000..46cc7b3
--- /dev/null
+++ b/tests/fixtures/layouts/dependent-github.yaml
@@ -0,0 +1,35 @@
+- pipeline:
+ name: gate
+ description: Gatekeeping
+ manager: dependent
+ trigger:
+ github:
+ - event: pull_request
+ action: labeled
+ label: 'merge'
+ success:
+ github:
+ merge: true
+ unlabel: 'merge'
+ failure:
+ github:
+ unlabel: 'merge'
+
+- job:
+ name: project-test1
+- job:
+ name: project-test2
+- job:
+ name: project-merge
+ failure-message: Unable to merge change
+ hold-following-changes: true
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
diff --git a/tests/fixtures/layouts/dequeue-github.yaml b/tests/fixtures/layouts/dequeue-github.yaml
new file mode 100644
index 0000000..25e92c9
--- /dev/null
+++ b/tests/fixtures/layouts/dequeue-github.yaml
@@ -0,0 +1,18 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action:
+ - opened
+ - changed
+
+- job:
+ name: one-job-project-merge
+
+- project:
+ name: org/one-job-project
+ check:
+ jobs:
+ - one-job-project-merge
diff --git a/tests/fixtures/layouts/files-github.yaml b/tests/fixtures/layouts/files-github.yaml
new file mode 100644
index 0000000..734b945
--- /dev/null
+++ b/tests/fixtures/layouts/files-github.yaml
@@ -0,0 +1,18 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: opened
+
+- job:
+ name: project-test1
+ files:
+ - '.*-requires'
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/layouts/ignore-dependencies.yaml b/tests/fixtures/layouts/ignore-dependencies.yaml
index 02aea36..86fe674 100644
--- a/tests/fixtures/layouts/ignore-dependencies.yaml
+++ b/tests/fixtures/layouts/ignore-dependencies.yaml
@@ -32,7 +32,6 @@
- job:
name: project1-project2-integration
- queue-name: integration
- project:
name: org/project1
diff --git a/tests/fixtures/layouts/labeling-github.yaml b/tests/fixtures/layouts/labeling-github.yaml
new file mode 100644
index 0000000..33ce993
--- /dev/null
+++ b/tests/fixtures/layouts/labeling-github.yaml
@@ -0,0 +1,29 @@
+- pipeline:
+ name: labels
+ description: Trigger on labels
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: labeled
+ label:
+ - 'test'
+ - event: pull_request
+ action: unlabeled
+ label:
+ - 'do not test'
+ success:
+ github:
+ label:
+ - 'tests passed'
+ unlabel:
+ - 'test'
+
+- job:
+ name: project-labels
+
+- project:
+ name: org/project
+ labels:
+ jobs:
+ - project-labels
diff --git a/tests/fixtures/layouts/live-reconfiguration-add-job.yaml b/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
new file mode 100644
index 0000000..5916282
--- /dev/null
+++ b/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
@@ -0,0 +1,57 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: project-merge
+ hold-following-changes: true
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- job:
+ name: project-test3
+
+- job:
+ name: project-testfile
+ files:
+ - '.*-requires'
+
+- project:
+ name: org/project
+ merge-mode: cherry-pick
+ gate:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies:
+ - project-merge
+ - project-test2:
+ dependencies:
+ - project-merge
+ - project-test3:
+ dependencies:
+ - project-merge
+ - project-testfile:
+ dependencies:
+ - project-merge
diff --git a/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml b/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
new file mode 100644
index 0000000..0907880
--- /dev/null
+++ b/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
@@ -0,0 +1,35 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: project-merge
+ hold-following-changes: true
+
+- job:
+ name: project-test2
+
+- job:
+ name: project-testfile
+
+- project:
+ name: org/project
+ merge-mode: cherry-pick
+ check:
+ jobs:
+ - project-merge
+ - project-test2:
+ dependencies:
+ - project-merge
+ - project-testfile:
+ dependencies:
+ - project-merge
diff --git a/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml b/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
new file mode 100644
index 0000000..bf4416a
--- /dev/null
+++ b/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
@@ -0,0 +1,86 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: project-merge
+ hold-following-changes: true
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- job:
+ name: project1-project2-integration
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ gate:
+ queue: integrated
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
+ gate:
+ queue: integrated
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
diff --git a/tests/fixtures/layouts/merging-github.yaml b/tests/fixtures/layouts/merging-github.yaml
new file mode 100644
index 0000000..4e13063
--- /dev/null
+++ b/tests/fixtures/layouts/merging-github.yaml
@@ -0,0 +1,19 @@
+- pipeline:
+ name: merge
+ description: Pipeline for merging the pull request
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: comment
+ comment: 'merge me'
+ success:
+ github:
+ merge: true
+ comment: false
+
+- project:
+ name: org/project
+ merge:
+ jobs:
+ - noop
diff --git a/tests/fixtures/layouts/push-tag-github.yaml b/tests/fixtures/layouts/push-tag-github.yaml
new file mode 100644
index 0000000..54683e9
--- /dev/null
+++ b/tests/fixtures/layouts/push-tag-github.yaml
@@ -0,0 +1,29 @@
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ github:
+ - event: push
+ ref: '^refs/heads/master$'
+
+- pipeline:
+ name: tag
+ manager: independent
+ trigger:
+ github:
+ - event: push
+ ref: ^refs/tags/.*$
+
+- job:
+ name: project-post
+- job:
+ name: project-tag
+
+- project:
+ name: org/project
+ post:
+ jobs:
+ - project-post
+ tag:
+ jobs:
+ - project-tag
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
new file mode 100644
index 0000000..bcbac1b
--- /dev/null
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -0,0 +1,45 @@
+- pipeline:
+ name: check
+ description: Standard check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: opened
+ start:
+ github:
+ status: 'pending'
+ comment: false
+ success:
+ github:
+ status: 'success'
+
+- pipeline:
+ name: reporting
+ description: Uncommon reporting
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: comment
+ comment: 'reporting check'
+ start:
+ github: {}
+ success:
+ github:
+ comment: false
+ failure:
+ github:
+ comment: false
+
+- job:
+ name: project-test1
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ reporting:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/layouts/reviews-github.yaml b/tests/fixtures/layouts/reviews-github.yaml
new file mode 100644
index 0000000..1cc887a
--- /dev/null
+++ b/tests/fixtures/layouts/reviews-github.yaml
@@ -0,0 +1,21 @@
+- pipeline:
+ name: reviews
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request_review
+ action: submitted
+ state: 'approve'
+ success:
+ github:
+ label:
+ - 'tests passed'
+
+- job:
+ name: project-reviews
+
+- project:
+ name: org/project
+ reviews:
+ jobs:
+ - project-reviews
diff --git a/tests/fixtures/zuul-connections-gerrit-and-github.conf b/tests/fixtures/zuul-connections-gerrit-and-github.conf
new file mode 100644
index 0000000..bd05c75
--- /dev/null
+++ b/tests/fixtures/zuul-connections-gerrit-and-github.conf
@@ -0,0 +1,31 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=config/multi-driver/main.yaml
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=none
+
+[connection github]
+driver=github
+
+[connection outgoing_smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf
new file mode 100644
index 0000000..ab34619
--- /dev/null
+++ b/tests/fixtures/zuul-github-driver.conf
@@ -0,0 +1,27 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+job_name_in_report=true
+status_url=http://zuul.example.com/status
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection github]
+driver=github
+
+[connection github_ssh]
+driver=github
+sshkey=/home/zuul/.ssh/id_rsa
+
+[connection github_ent]
+driver=github
+sshkey=/home/zuul/.ssh/id_rsa
+git_host=github.enterprise.io
diff --git a/tests/unit/test_change_matcher.py b/tests/unit/test_change_matcher.py
index 0585322..6b161a1 100644
--- a/tests/unit/test_change_matcher.py
+++ b/tests/unit/test_change_matcher.py
@@ -125,12 +125,18 @@
def test_matches_returns_false_when_not_all_files_match(self):
self._test_matches(False, files=['/COMMIT_MSG', 'docs/foo', 'foo/bar'])
+ def test_matches_returns_true_when_single_file_does_not_match(self):
+ self._test_matches(True, files=['docs/foo'])
+
def test_matches_returns_false_when_commit_message_matches(self):
self._test_matches(False, files=['/COMMIT_MSG'])
def test_matches_returns_true_when_all_files_match(self):
self._test_matches(True, files=['/COMMIT_MSG', 'docs/foo'])
+ def test_matches_returns_true_when_single_file_matches(self):
+ self._test_matches(True, files=['docs/foo'])
+
class TestMatchAll(BaseTestMatcher):
diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py
index 4dda78b..b424769 100644
--- a/tests/unit/test_encryption.py
+++ b/tests/unit/test_encryption.py
@@ -41,14 +41,14 @@
def test_pkcs1_oaep(self):
"Verify encryption and decryption"
- orig_plaintext = "some text to encrypt"
+ orig_plaintext = b"some text to encrypt"
ciphertext = encryption.encrypt_pkcs1_oaep(orig_plaintext, self.public)
plaintext = encryption.decrypt_pkcs1_oaep(ciphertext, self.private)
self.assertEqual(orig_plaintext, plaintext)
def test_openssl_pkcs1_oaep(self):
"Verify that we can decrypt something encrypted with OpenSSL"
- orig_plaintext = "some text to encrypt"
+ orig_plaintext = b"some text to encrypt"
pem_public = encryption.serialize_rsa_public_key(self.public)
public_file = tempfile.NamedTemporaryFile(delete=False)
try:
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
new file mode 100644
index 0000000..f918218
--- /dev/null
+++ b/tests/unit/test_github_driver.py
@@ -0,0 +1,498 @@
+# Copyright 2015 GoodData
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+from testtools.matchers import MatchesRegex
+import time
+
+from tests.base import ZuulTestCase, simple_layout, random_sha1
+
+
+class TestGithubDriver(ZuulTestCase):
+ config_file = 'zuul-github-driver.conf'
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_pull_event(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+
+ build_params = self.builds[0].parameters
+ self.assertEqual('master', build_params['ZUUL_BRANCH'])
+ self.assertEqual(str(A.number), build_params['ZUUL_CHANGE'])
+ self.assertEqual(A.head_sha, build_params['ZUUL_PATCHSET'])
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-test1').result)
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-test2').result)
+
+ job = self.getJobFromHistory('project-test2')
+ zuulvars = job.parameters['vars']['zuul']
+ self.assertEqual(A.number, zuulvars['change'])
+ self.assertEqual(A.head_sha, zuulvars['patchset'])
+ self.assertEqual(1, len(A.comments))
+ self.assertEqual(2, len(self.history))
+
+ # test_pull_unmatched_branch_event(self):
+ self.create_branch('org/project', 'unmatched_branch')
+ B = self.fake_github.openFakePullRequest(
+ 'org/project', 'unmatched_branch', 'B')
+ self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+
+ self.assertEqual(2, len(self.history))
+
+ @simple_layout('layouts/files-github.yaml', driver='github')
+ def test_pull_matched_file_event(self):
+ A = self.fake_github.openFakePullRequest(
+ 'org/project', 'master', 'A',
+ files=['random.txt', 'build-requires'])
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(1, len(self.history))
+
+ # test_pull_unmatched_file_event
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B',
+ files=['random.txt'])
+ self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(1, len(self.history))
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_comment_event(self):
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getCommentAddedEvent('test me'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+
+ # Test an unmatched comment, history should remain the same
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ self.fake_github.emitEvent(B.getCommentAddedEvent('casual comment'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+
+ @simple_layout('layouts/push-tag-github.yaml', driver='github')
+ def test_tag_event(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ sha = random_sha1()
+ self.fake_github.emitEvent(
+ self.fake_github.getPushEvent('org/project', 'refs/tags/newtag',
+ new_rev=sha))
+ self.waitUntilSettled()
+
+ build_params = self.builds[0].parameters
+ self.assertEqual('refs/tags/newtag', build_params['ZUUL_REF'])
+ self.assertEqual('00000000000000000000000000000000',
+ build_params['ZUUL_OLDREV'])
+ self.assertEqual(sha, build_params['ZUUL_NEWREV'])
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-tag').result)
+
+ @simple_layout('layouts/push-tag-github.yaml', driver='github')
+ def test_push_event(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ old_sha = random_sha1()
+ new_sha = random_sha1()
+ self.fake_github.emitEvent(
+ self.fake_github.getPushEvent('org/project', 'refs/heads/master',
+ old_sha, new_sha))
+ self.waitUntilSettled()
+
+ build_params = self.builds[0].parameters
+ self.assertEqual('refs/heads/master', build_params['ZUUL_REF'])
+ self.assertEqual(old_sha, build_params['ZUUL_OLDREV'])
+ self.assertEqual(new_sha, build_params['ZUUL_NEWREV'])
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-post').result)
+ self.assertEqual(1, len(self.history))
+
+ # test unmatched push event
+ old_sha = random_sha1()
+ new_sha = random_sha1()
+ self.fake_github.emitEvent(
+ self.fake_github.getPushEvent('org/project',
+ 'refs/heads/unmatched_branch',
+ old_sha, new_sha))
+ self.waitUntilSettled()
+
+ self.assertEqual(1, len(self.history))
+
+ @simple_layout('layouts/labeling-github.yaml', driver='github')
+ def test_labels(self):
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.addLabel('test'))
+ self.waitUntilSettled()
+ self.assertEqual(1, len(self.history))
+ self.assertEqual('project-labels', self.history[0].name)
+ self.assertEqual(['tests passed'], A.labels)
+
+ # test label removed
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ B.addLabel('do not test')
+ self.fake_github.emitEvent(B.removeLabel('do not test'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+ self.assertEqual('project-labels', self.history[1].name)
+ self.assertEqual(['tests passed'], B.labels)
+
+ # test unmatched label
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+ self.fake_github.emitEvent(C.addLabel('other label'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+ self.assertEqual(['other label'], C.labels)
+
+ @simple_layout('layouts/reviews-github.yaml', driver='github')
+ def test_review_event(self):
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getReviewAddedEvent('approve'))
+ self.waitUntilSettled()
+ self.assertEqual(1, len(self.history))
+ self.assertEqual('project-reviews', self.history[0].name)
+ self.assertEqual(['tests passed'], A.labels)
+
+ # test_review_unmatched_event
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ self.fake_github.emitEvent(B.getReviewAddedEvent('comment'))
+ self.waitUntilSettled()
+ self.assertEqual(1, len(self.history))
+
+ @simple_layout('layouts/dequeue-github.yaml', driver='github')
+ def test_dequeue_pull_synchronized(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest(
+ 'org/one-job-project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+
+ # event update stamp has resolution one second, wait so the latter
+ # one has newer timestamp
+ time.sleep(1)
+ A.addCommit()
+ self.fake_github.emitEvent(A.getPullRequestSynchronizeEvent())
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(2, len(self.history))
+ self.assertEqual(1, self.countJobResults(self.history, 'ABORTED'))
+
+ @simple_layout('layouts/dequeue-github.yaml', driver='github')
+ def test_dequeue_pull_abandoned(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest(
+ 'org/one-job-project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+ self.fake_github.emitEvent(A.getPullRequestClosedEvent())
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(1, len(self.history))
+ self.assertEqual(1, self.countJobResults(self.history, 'ABORTED'))
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_git_https_url(self):
+ """Test that git_ssh option gives git url with ssh"""
+ url = self.fake_github.real_getGitUrl('org/project')
+ self.assertEqual('https://github.com/org/project', url)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_git_ssh_url(self):
+ """Test that git_ssh option gives git url with ssh"""
+ url = self.fake_github_ssh.real_getGitUrl('org/project')
+ self.assertEqual('ssh://git@github.com/org/project.git', url)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_git_enterprise_url(self):
+ """Test that git_url option gives git url with proper host"""
+ url = self.fake_github_ent.real_getGitUrl('org/project')
+ self.assertEqual('ssh://git@github.enterprise.io/org/project.git', url)
+
+ @simple_layout('layouts/reporting-github.yaml', driver='github')
+ def test_reporting(self):
+ # pipeline reports pull status both on start and success
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+ self.assertIn('check', A.statuses)
+ check_status = A.statuses['check']
+ check_url = ('http://zuul.example.com/status/#%s,%s' %
+ (A.number, A.head_sha))
+ self.assertEqual('Standard check', check_status['description'])
+ self.assertEqual('pending', check_status['state'])
+ self.assertEqual(check_url, check_status['url'])
+ self.assertEqual(0, len(A.comments))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ check_status = A.statuses['check']
+ self.assertEqual('Standard check', check_status['description'])
+ self.assertEqual('success', check_status['state'])
+ self.assertEqual(check_url, check_status['url'])
+ self.assertEqual(1, len(A.comments))
+ self.assertThat(A.comments[0],
+ MatchesRegex('.*Build succeeded.*', re.DOTALL))
+
+ # pipeline does not report any status but does comment
+ self.executor_server.hold_jobs_in_build = True
+ self.fake_github.emitEvent(
+ A.getCommentAddedEvent('reporting check'))
+ self.waitUntilSettled()
+ self.assertNotIn('reporting', A.statuses)
+ # comments increased by one for the start message
+ self.assertEqual(2, len(A.comments))
+ self.assertThat(A.comments[1],
+ MatchesRegex('.*Starting reporting jobs.*', re.DOTALL))
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertNotIn('reporting', A.statuses)
+ self.assertEqual(2, len(A.comments))
+
+ @simple_layout('layouts/merging-github.yaml', driver='github')
+ def test_report_pull_merge(self):
+ # pipeline merges the pull request on success
+ A = self.fake_github.openFakePullRequest('org/project', 'master',
+ 'PR title')
+ self.fake_github.emitEvent(A.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertTrue(A.is_merged)
+ self.assertThat(A.merge_message,
+ MatchesRegex('.*PR title.*Reviewed-by.*', re.DOTALL))
+
+ # pipeline merges the pull request on success after failure
+ self.fake_github.merge_failure = True
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ self.fake_github.emitEvent(B.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertFalse(B.is_merged)
+ self.fake_github.merge_failure = False
+
+ # pipeline merges the pull request on second run of merge
+ # first merge failed on 405 Method Not Allowed error
+ self.fake_github.merge_not_allowed_count = 1
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+ self.fake_github.emitEvent(C.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertTrue(C.is_merged)
+
+ # pipeline does not merge the pull request
+ # merge failed on 405 Method Not Allowed error - twice
+ self.fake_github.merge_not_allowed_count = 2
+ D = self.fake_github.openFakePullRequest('org/project', 'master', 'D')
+ self.fake_github.emitEvent(D.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertFalse(D.is_merged)
+
+ @simple_layout('layouts/dependent-github.yaml', driver='github')
+ def test_parallel_changes(self):
+ "Test that changes are tested in parallel and merged in series"
+
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+
+ self.fake_github.emitEvent(A.addLabel('merge'))
+ self.fake_github.emitEvent(B.addLabel('merge'))
+ self.fake_github.emitEvent(C.addLabel('merge'))
+
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 1)
+ self.assertEqual(self.builds[0].name, 'project-merge')
+ self.assertTrue(self.builds[0].hasChanges(A))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 3)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertTrue(self.builds[1].hasChanges(A))
+ self.assertEqual(self.builds[2].name, 'project-merge')
+ self.assertTrue(self.builds[2].hasChanges(A, B))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 5)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertTrue(self.builds[1].hasChanges(A))
+
+ self.assertEqual(self.builds[2].name, 'project-test1')
+ self.assertTrue(self.builds[2].hasChanges(A))
+ self.assertEqual(self.builds[3].name, 'project-test2')
+ self.assertTrue(self.builds[3].hasChanges(A, B))
+
+ self.assertEqual(self.builds[4].name, 'project-merge')
+ self.assertTrue(self.builds[4].hasChanges(A, B, C))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 6)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertTrue(self.builds[1].hasChanges(A))
+
+ self.assertEqual(self.builds[2].name, 'project-test1')
+ self.assertTrue(self.builds[2].hasChanges(A, B))
+ self.assertEqual(self.builds[3].name, 'project-test2')
+ self.assertTrue(self.builds[3].hasChanges(A, B))
+
+ self.assertEqual(self.builds[4].name, 'project-test1')
+ self.assertTrue(self.builds[4].hasChanges(A, B, C))
+ self.assertEqual(self.builds[5].name, 'project-test2')
+ self.assertTrue(self.builds[5].hasChanges(A, B, C))
+
+ all_builds = self.builds[:]
+ self.release(all_builds[2])
+ self.release(all_builds[3])
+ self.waitUntilSettled()
+ self.assertFalse(A.is_merged)
+ self.assertFalse(B.is_merged)
+ self.assertFalse(C.is_merged)
+
+ self.release(all_builds[0])
+ self.release(all_builds[1])
+ self.waitUntilSettled()
+ self.assertTrue(A.is_merged)
+ self.assertTrue(B.is_merged)
+ self.assertFalse(C.is_merged)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), 9)
+ self.assertTrue(C.is_merged)
+
+ self.assertNotIn('merge', A.labels)
+ self.assertNotIn('merge', B.labels)
+ self.assertNotIn('merge', C.labels)
+
+ @simple_layout('layouts/dependent-github.yaml', driver='github')
+ def test_failed_changes(self):
+ "Test that a change behind a failed change is retested"
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+
+ self.executor_server.failJob('project-test1', A)
+
+ self.fake_github.emitEvent(A.addLabel('merge'))
+ self.fake_github.emitEvent(B.addLabel('merge'))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+
+ self.waitUntilSettled()
+ # It's certain that the merge job for change 2 will run, but
+ # the test1 and test2 jobs may or may not run.
+ self.assertTrue(len(self.history) > 6)
+ self.assertFalse(A.is_merged)
+ self.assertTrue(B.is_merged)
+ self.assertNotIn('merge', A.labels)
+ self.assertNotIn('merge', B.labels)
+
+ @simple_layout('layouts/dependent-github.yaml', driver='github')
+ def test_failed_change_at_head(self):
+ "Test that if a change at the head fails, jobs behind it are canceled"
+
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+
+ self.executor_server.failJob('project-test1', A)
+
+ self.fake_github.emitEvent(A.addLabel('merge'))
+ self.fake_github.emitEvent(B.addLabel('merge'))
+ self.fake_github.emitEvent(C.addLabel('merge'))
+
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 1)
+ self.assertEqual(self.builds[0].name, 'project-merge')
+ self.assertTrue(self.builds[0].hasChanges(A))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 6)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertEqual(self.builds[2].name, 'project-test1')
+ self.assertEqual(self.builds[3].name, 'project-test2')
+ self.assertEqual(self.builds[4].name, 'project-test1')
+ self.assertEqual(self.builds[5].name, 'project-test2')
+
+ self.release(self.builds[0])
+ self.waitUntilSettled()
+
+ # project-test2, project-merge for B
+ self.assertEqual(len(self.builds), 2)
+ self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), 15)
+ self.assertFalse(A.is_merged)
+ self.assertTrue(B.is_merged)
+ self.assertTrue(C.is_merged)
+ self.assertNotIn('merge', A.labels)
+ self.assertNotIn('merge', B.labels)
+ self.assertNotIn('merge', C.labels)
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index d8480ea..5f968b4 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -73,11 +73,21 @@
change.files = ['/COMMIT_MSG', 'docs/foo']
self.assertFalse(self.job.changeMatches(change))
+ def test_change_matches_returns_false_for_single_matched_skip_if(self):
+ change = model.Change('project')
+ change.files = ['docs/foo']
+ self.assertFalse(self.job.changeMatches(change))
+
def test_change_matches_returns_true_for_unmatched_skip_if(self):
change = model.Change('project')
change.files = ['/COMMIT_MSG', 'foo']
self.assertTrue(self.job.changeMatches(change))
+ def test_change_matches_returns_true_for_single_unmatched_skip_if(self):
+ change = model.Change('project')
+ change.files = ['foo']
+ self.assertTrue(self.job.changeMatches(change))
+
def test_job_sets_defaults_for_boolean_attributes(self):
self.assertIsNotNone(self.job.voting)
diff --git a/tests/unit/test_multi_driver.py b/tests/unit/test_multi_driver.py
new file mode 100644
index 0000000..a1107de
--- /dev/null
+++ b/tests/unit/test_multi_driver.py
@@ -0,0 +1,45 @@
+# Copyright 2015 GoodData
+# Copyright (c) 2017 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tests.base import ZuulTestCase
+
+
+class TestGerritAndGithub(ZuulTestCase):
+ config_file = 'zuul-connections-gerrit-and-github.conf'
+ tenant_config_file = 'config/multi-driver/main.yaml'
+
+ def setup_config(self):
+ super(TestGerritAndGithub, self).setup_config()
+
+ def test_multiple_project_gerrit_and_github(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+
+ B = self.fake_github.openFakePullRequest('org/project1', 'master', 'B')
+ self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
+
+ self.waitUntilSettled()
+
+ self.assertEqual(2, len(self.builds))
+ self.assertEqual('project-gerrit', self.builds[0].name)
+ self.assertEqual('project1-github', self.builds[1].name)
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertTrue(self.builds[1].hasChanges(B))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index f67318d..f394c0c 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -21,8 +21,9 @@
import os
import re
import shutil
+import sys
import time
-from unittest import skip
+from unittest import (skip, skipIf)
import git
from six.moves import urllib
@@ -509,6 +510,7 @@
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_failed_change_at_head_with_queue(self):
"Test that if a change at the head fails, queued jobs are canceled"
@@ -530,8 +532,8 @@
queue = self.gearman_server.getQueue()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(queue), 1)
- self.assertEqual(queue[0].name, 'executor:execute')
- job_args = json.loads(queue[0].arguments)
+ self.assertEqual(queue[0].name, b'executor:execute')
+ job_args = json.loads(queue[0].arguments.decode('utf8'))
self.assertEqual(job_args['job'], 'project-merge')
self.assertEqual(job_args['items'][0]['number'], '%d' % A.number)
@@ -547,17 +549,23 @@
self.assertEqual(len(queue), 6)
self.assertEqual(
- json.loads(queue[0].arguments)['job'], 'project-test1')
+ json.loads(queue[0].arguments.decode('utf8'))['job'],
+ 'project-test1')
self.assertEqual(
- json.loads(queue[1].arguments)['job'], 'project-test2')
+ json.loads(queue[1].arguments.decode('utf8'))['job'],
+ 'project-test2')
self.assertEqual(
- json.loads(queue[2].arguments)['job'], 'project-test1')
+ json.loads(queue[2].arguments.decode('utf8'))['job'],
+ 'project-test1')
self.assertEqual(
- json.loads(queue[3].arguments)['job'], 'project-test2')
+ json.loads(queue[3].arguments.decode('utf8'))['job'],
+ 'project-test2')
self.assertEqual(
- json.loads(queue[4].arguments)['job'], 'project-test1')
+ json.loads(queue[4].arguments.decode('utf8'))['job'],
+ 'project-test1')
self.assertEqual(
- json.loads(queue[5].arguments)['job'], 'project-test2')
+ json.loads(queue[5].arguments.decode('utf8'))['job'],
+ 'project-test2')
self.release(queue[0])
self.waitUntilSettled()
@@ -929,6 +937,7 @@
a = source.getChange(event, refresh=True)
self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_project_merge_conflict(self):
"Test that gate merge conflicts are handled properly"
@@ -980,6 +989,7 @@
dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
], ordered=False)
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_delayed_merge_conflict(self):
"Test that delayed check merge conflicts are handled properly"
@@ -1916,6 +1926,7 @@
self.assertEqual(A.reported, 2)
@simple_layout('layouts/no-jobs-project.yaml')
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_no_job_project(self):
"Test that reports with no jobs don't get sent"
A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
@@ -2047,6 +2058,7 @@
self.assertReportedStat('test-timing', '3|ms')
self.assertReportedStat('test-gauge', '12|g')
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_stuck_job_cleanup(self):
"Test that pending jobs are cleaned up if removed from layout"
@@ -2164,12 +2176,6 @@
self.assertEqual(set(['project-test-nomatch-starts-empty',
'project-test-nomatch-starts-full']), run_jobs)
- @skip("Disabled for early v3 development")
- def test_test_config(self):
- "Test that we can test the config"
- self.sched.testConfig(self.config.get('zuul', 'tenant_config'),
- self.connections)
-
def test_queue_names(self):
"Test shared change queue names"
tenant = self.sched.abide.tenants.get('tenant-one')
@@ -2180,6 +2186,7 @@
self.assertEqual(q1.name, 'integrated')
self.assertEqual(q2.name, 'integrated')
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_queue_precedence(self):
"Test that queue precedence works"
@@ -2233,7 +2240,7 @@
self.assertIn('Cache-Control', headers)
self.assertIn('Last-Modified', headers)
self.assertIn('Expires', headers)
- data = f.read()
+ data = f.read().decode('utf8')
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
@@ -2293,13 +2300,11 @@
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
- @skip("Disabled for early v3 development")
def test_live_reconfiguration_merge_conflict(self):
# A real-world bug: a change in a gate queue has a merge
# conflict and a job is added to its project while it's
# sitting in the queue. The job gets added to the change and
# enqueued and the change gets stuck.
- self.worker.registerFunction('build:project-test3')
self.executor_server.hold_jobs_in_build = True
# This change is fine. It's here to stop the queue long
@@ -2307,14 +2312,14 @@
# reconfiguration, as well as to provide a conflict for the
# next change. This change will succeed and merge.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
- A.addPatchset(['conflict'])
+ A.addPatchset({'conflict': 'A'})
A.addApproval('code-review', 2)
# This change will be in merge conflict. During the
# reconfiguration, we will add a job. We want to make sure
# that doesn't cause it to get stuck.
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
- B.addPatchset(['conflict'])
+ B.addPatchset({'conflict': 'B'})
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
@@ -2330,8 +2335,8 @@
self.assertEqual(len(self.history), 0)
# Add the "project-test3" job.
- self.updateConfigLayout(
- 'tests/fixtures/layout-live-reconfiguration-add-job.yaml')
+ self.commitConfigUpdate('common-config',
+ 'layouts/live-reconfiguration-add-job.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
@@ -2353,19 +2358,17 @@
'SUCCESS')
self.assertEqual(len(self.history), 4)
- @skip("Disabled for early v3 development")
def test_live_reconfiguration_failed_root(self):
# An extrapolation of test_live_reconfiguration_merge_conflict
# that tests a job added to a job tree with a failed root does
# not run.
- self.worker.registerFunction('build:project-test3')
self.executor_server.hold_jobs_in_build = True
# This change is fine. It's here to stop the queue long
# enough for the next change to be subject to the
# reconfiguration. This change will succeed and merge.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
- A.addPatchset(['conflict'])
+ A.addPatchset({'conflict': 'A'})
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
@@ -2393,8 +2396,8 @@
self.assertEqual(len(self.history), 2)
# Add the "project-test3" job.
- self.updateConfigLayout(
- 'tests/fixtures/layout-live-reconfiguration-add-job.yaml')
+ self.commitConfigUpdate('common-config',
+ 'layouts/live-reconfiguration-add-job.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
@@ -2415,7 +2418,6 @@
self.assertEqual(self.history[4].result, 'SUCCESS')
self.assertEqual(len(self.history), 5)
- @skip("Disabled for early v3 development")
def test_live_reconfiguration_failed_job(self):
# Test that a change with a removed failing job does not
# disrupt reconfiguration. If a change has a failed job and
@@ -2447,8 +2449,8 @@
self.assertEqual(len(self.history), 2)
# Remove the test1 job.
- self.updateConfigLayout(
- 'tests/fixtures/layout-live-reconfiguration-failed-job.yaml')
+ self.commitConfigUpdate('common-config',
+ 'layouts/live-reconfiguration-failed-job.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
@@ -2468,7 +2470,6 @@
# Ensure the removed job was not included in the report.
self.assertNotIn('project-test1', A.messages[0])
- @skip("Disabled for early v3 development")
def test_live_reconfiguration_shared_queue(self):
# Test that a change with a failing job which was removed from
# this project but otherwise still exists in the system does
@@ -2490,15 +2491,16 @@
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 0)
- self.assertEqual(self.getJobFromHistory('project1-merge').result,
+ self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project1-project2-integration').result, 'FAILURE')
self.assertEqual(len(self.history), 2)
# Remove the integration job.
- self.updateConfigLayout(
- 'tests/fixtures/layout-live-reconfiguration-shared-queue.yaml')
+ self.commitConfigUpdate(
+ 'common-config',
+ 'layouts/live-reconfiguration-shared-queue.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
@@ -2506,11 +2508,11 @@
self.executor_server.release()
self.waitUntilSettled()
- self.assertEqual(self.getJobFromHistory('project1-merge').result,
+ self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
- self.assertEqual(self.getJobFromHistory('project1-test1').result,
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
- self.assertEqual(self.getJobFromHistory('project1-test2').result,
+ self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project1-project2-integration').result, 'FAILURE')
@@ -2522,7 +2524,6 @@
# Ensure the removed job was not included in the report.
self.assertNotIn('project1-project2-integration', A.messages[0])
- @skip("Disabled for early v3 development")
def test_double_live_reconfiguration_shared_queue(self):
# This was a real-world regression. A change is added to
# gate; a reconfigure happens, a second change which depends
@@ -2724,7 +2725,7 @@
req = urllib.request.Request(
"http://localhost:%s/tenant-one/status" % port)
f = urllib.request.urlopen(req)
- data = f.read()
+ data = f.read().decode('utf8')
self.executor_server.hold_jobs_in_build = False
# Stop queuing timer triggered jobs so that the assertions
@@ -3351,7 +3352,7 @@
if time.time() - start > 10:
raise Exception("Timeout waiting for gearman server to report "
+ "back to the client")
- build = self.executor.builds.values()[0]
+ build = list(self.executor.builds.values())[0]
if build.worker.name == "My Worker":
break
else:
@@ -3524,7 +3525,7 @@
if time.time() - start > 10:
raise Exception("Timeout waiting for gearman server to report "
+ "back to the client")
- build = self.executor_client.builds.values()[0]
+ build = list(self.executor_client.builds.values())[0]
if build.worker.name == "My Worker":
break
else:
@@ -3872,6 +3873,7 @@
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 0)
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_crd_check(self):
"Test cross-repo dependencies in independent pipelines"
@@ -4022,9 +4024,11 @@
self.assertEqual(self.history[0].changes, '2,1 1,1')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_crd_check_reconfiguration(self):
self._test_crd_check_reconfiguration('org/project1', 'org/project2')
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_crd_undefined_project(self):
"""Test that undefined projects in dependencies are handled for
independent pipelines"""
@@ -4034,6 +4038,7 @@
self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
@simple_layout('layouts/ignore-dependencies.yaml')
+ @skipIf(sys.version_info.major > 2, 'Fails on py3')
def test_crd_check_ignore_dependencies(self):
"Test cross-repo dependencies can be ignored"
diff --git a/tests/unit/test_scheduler_cmd.py b/tests/unit/test_scheduler_cmd.py
new file mode 100644
index 0000000..ee6200f
--- /dev/null
+++ b/tests/unit/test_scheduler_cmd.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import testtools
+import zuul.cmd.scheduler
+
+from tests import base
+
+
+class TestSchedulerCmdArguments(testtools.TestCase):
+
+ def setUp(self):
+ super(TestSchedulerCmdArguments, self).setUp()
+ self.app = zuul.cmd.scheduler.Scheduler()
+
+ def test_test_config(self):
+ conf_path = os.path.join(base.FIXTURE_DIR, 'zuul.conf')
+ self.app.parse_arguments(['-t', '-c', conf_path])
+ self.assertTrue(self.app.args.validate)
+ self.app.read_config()
+ self.assertEqual(0, self.app.test_config())
diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py
index 4511ec7..b2836ae 100644
--- a/tests/unit/test_webapp.py
+++ b/tests/unit/test_webapp.py
@@ -51,7 +51,7 @@
req = urllib.request.Request(
"http://localhost:%s/tenant-one/status" % self.port)
f = urllib.request.urlopen(req)
- data = json.loads(f.read())
+ data = json.loads(f.read().decode('utf8'))
self.assertIn('pipelines', data)
@@ -60,7 +60,7 @@
req = urllib.request.Request(
"http://localhost:%s/tenant-one/status.json" % self.port)
f = urllib.request.urlopen(req)
- data = json.loads(f.read())
+ data = json.loads(f.read().decode('utf8'))
self.assertIn('pipelines', data)
@@ -75,7 +75,7 @@
req = urllib.request.Request(
"http://localhost:%s/tenant-one/status/change/1,1" % self.port)
f = urllib.request.urlopen(req)
- data = json.loads(f.read())
+ data = json.loads(f.read().decode('utf8'))
self.assertEqual(1, len(data), data)
self.assertEqual("org/project", data[0]['project'])
@@ -83,13 +83,13 @@
req = urllib.request.Request(
"http://localhost:%s/tenant-one/status/change/2,1" % self.port)
f = urllib.request.urlopen(req)
- data = json.loads(f.read())
+ data = json.loads(f.read().decode('utf8'))
self.assertEqual(1, len(data), data)
self.assertEqual("org/project1", data[0]['project'], data)
def test_webapp_keys(self):
- with open(os.path.join(FIXTURE_DIR, 'public.pem')) as f:
+ with open(os.path.join(FIXTURE_DIR, 'public.pem'), 'rb') as f:
public_pem = f.read()
req = urllib.request.Request(
@@ -106,7 +106,7 @@
req = urllib.request.Request(
"http://localhost:%s/custom" % self.port)
f = urllib.request.urlopen(req)
- self.assertEqual('ok', f.read())
+ self.assertEqual(b'ok', f.read())
self.webapp.unregister_path('/custom')
self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, req)
diff --git a/tox.ini b/tox.ini
index 8235483..6a50c6d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,6 @@
[flake8]
# These are ignored intentionally in openstack-infra projects;
# please don't submit patches that solely correct them or enable them.
-ignore = E125,E129,H
+ignore = E305,E125,E129,E402,H,W503
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg
diff --git a/zuul/ansible/library/command.py b/zuul/ansible/library/command.py
index 328ae7b..52de5a4 100644
--- a/zuul/ansible/library/command.py
+++ b/zuul/ansible/library/command.py
@@ -123,6 +123,8 @@
LOG_STREAM_FILE = '/tmp/console.log'
PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?')
+# List to save stdout log lines in as we collect them
+_log_lines = []
class Console(object):
@@ -150,6 +152,7 @@
line = fd.readline()
if not line:
break
+ _log_lines.append(line)
if not line.endswith('\n'):
line += '\n'
newline_warning = True
@@ -330,7 +333,8 @@
# cmd.stdout.close()
# ZUUL: stdout and stderr are in the console log file
- stdout = ''
+ # ZUUL: return the saved log lines so we can ship them back
+ stdout = ''.join(_log_lines)
stderr = ''
rc = cmd.returncode
diff --git a/zuul/ansible/library/zuul_afs.py b/zuul/ansible/library/zuul_afs.py
index 3ba426b..710c15d 100644
--- a/zuul/ansible/library/zuul_afs.py
+++ b/zuul/ansible/library/zuul_afs.py
@@ -116,6 +116,7 @@
module.exit_json(changed=True, build_roots=output)
from ansible.module_utils.basic import * # noqa
+from ansible.module_utils.basic import AnsibleModule
if __name__ == '__main__':
main()
diff --git a/zuul/ansible/library/zuul_console.py b/zuul/ansible/library/zuul_console.py
index 1932cf9..b1dc2d9 100644
--- a/zuul/ansible/library/zuul_console.py
+++ b/zuul/ansible/library/zuul_console.py
@@ -17,8 +17,10 @@
import os
import sys
+import select
import socket
import threading
+import time
LOG_STREAM_FILE = '/tmp/console.log'
LOG_STREAM_PORT = 19885
@@ -181,6 +183,7 @@
s.run()
from ansible.module_utils.basic import * # noqa
+from ansible.module_utils.basic import AnsibleModule
if __name__ == '__main__':
main()
diff --git a/zuul/change_matcher.py b/zuul/change_matcher.py
index 1da1d2c..baea217 100644
--- a/zuul/change_matcher.py
+++ b/zuul/change_matcher.py
@@ -108,7 +108,9 @@
yield self.commit_regex
def matches(self, change):
- if not (hasattr(change, 'files') and len(change.files) > 1):
+ if not (hasattr(change, 'files') and change.files):
+ return False
+ if len(change.files) == 1 and self.commit_regex.match(change.files[0]):
return False
for file_ in change.files:
matched_file = False
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
old mode 100644
new mode 100755
index f2a2612..d31c5b8
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -98,6 +98,6 @@
else:
logging.basicConfig(level=logging.DEBUG)
- def configure_connections(self):
+ def configure_connections(self, source_only=False):
self.connections = zuul.lib.connections.ConnectionRegistry()
- self.connections.configure(self.config)
+ self.connections.configure(self.config, source_only)
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
old mode 100644
new mode 100755
index 96ba4b3..1893f5a
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -106,7 +106,7 @@
server.send_command(server.args.command)
sys.exit(0)
- server.configure_connections()
+ server.configure_connections(source_only=True)
if server.config.has_option('executor', 'pidfile'):
pid_fn = os.path.expanduser(server.config.get('executor', 'pidfile'))
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
old mode 100644
new mode 100755
index 797a990..686f34a
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -77,7 +77,7 @@
server.parse_arguments()
server.read_config()
- server.configure_connections()
+ server.configure_connections(source_only=True)
if server.config.has_option('zuul', 'state_dir'):
state_dir = os.path.expanduser(server.config.get('zuul', 'state_dir'))
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index f1d1015..5328bba 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -40,7 +40,7 @@
super(Scheduler, self).__init__()
self.gear_server_pid = None
- def parse_arguments(self):
+ def parse_arguments(self, args=None):
parser = argparse.ArgumentParser(description='Project gating system.')
parser.add_argument('-c', dest='config',
help='specify the config file')
@@ -52,7 +52,7 @@
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
- self.args = parser.parse_args()
+ self.args = parser.parse_args(args)
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 070e731..90440f7 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import base64
from contextlib import contextmanager
import copy
import os
@@ -111,7 +112,7 @@
r = super(ZuulSafeLoader, self).construct_mapping(node, deep)
keys = frozenset(r.keys())
if len(keys) == 1 and keys.intersection(self.zuul_node_types):
- d = r.values()[0]
+ d = list(r.values())[0]
if isinstance(d, dict):
d['_start_mark'] = node.start_mark
d['_source_context'] = self.zuul_context
@@ -142,7 +143,7 @@
yaml_loader = yaml.SafeLoader
def __init__(self, ciphertext):
- self.ciphertext = ciphertext.decode('base64')
+ self.ciphertext = base64.b64decode(ciphertext)
def __ne__(self, other):
return not self.__eq__(other)
@@ -157,7 +158,8 @@
return cls(node.value)
def decrypt(self, private_key):
- return encryption.decrypt_pkcs1_oaep(self.ciphertext, private_key)
+ return encryption.decrypt_pkcs1_oaep(self.ciphertext,
+ private_key).decode('utf8')
class NodeSetParser(object):
@@ -229,7 +231,6 @@
job = {vs.Required('name'): str,
'parent': str,
- 'queue-name': str,
'failure-message': str,
'success-message': str,
'failure-url': str,
@@ -493,7 +494,7 @@
attrs = dict(name=conf_job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params
- jobname, attrs = conf_job.items()[0]
+ jobname, attrs = list(conf_job.items())[0]
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
diff --git a/zuul/driver/gerrit/__init__.py b/zuul/driver/gerrit/__init__.py
index 3bc371e..a36e912 100644
--- a/zuul/driver/gerrit/__init__.py
+++ b/zuul/driver/gerrit/__init__.py
@@ -14,10 +14,10 @@
from zuul.driver import Driver, ConnectionInterface, TriggerInterface
from zuul.driver import SourceInterface, ReporterInterface
-import gerritconnection
-import gerrittrigger
-import gerritsource
-import gerritreporter
+from zuul.driver.gerrit import gerritconnection
+from zuul.driver.gerrit import gerrittrigger
+from zuul.driver.gerrit import gerritsource
+from zuul.driver.gerrit import gerritreporter
class GerritDriver(Driver, ConnectionInterface, TriggerInterface,
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 73979be..a70662b 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -171,11 +171,15 @@
self._stopped = False
def _read(self, fd):
- l = fd.readline()
- data = json.loads(l)
- self.log.debug("Received data from Gerrit event stream: \n%s" %
- pprint.pformat(data))
- self.gerrit_connection.addEvent(data)
+ while True:
+ l = fd.readline()
+ data = json.loads(l)
+ self.log.debug("Received data from Gerrit event stream: \n%s" %
+ pprint.pformat(data))
+ self.gerrit_connection.addEvent(data)
+ # Continue until all the lines received are consumed
+ if fd._pos == fd._realpos:
+ break
def _listen(self, stdout, stderr):
poll = select.poll()
diff --git a/zuul/driver/git/__init__.py b/zuul/driver/git/__init__.py
index abedf6a..5ebedac 100644
--- a/zuul/driver/git/__init__.py
+++ b/zuul/driver/git/__init__.py
@@ -13,8 +13,8 @@
# under the License.
from zuul.driver import Driver, ConnectionInterface, SourceInterface
-import gitconnection
-import gitsource
+from zuul.driver.git import gitconnection
+from zuul.driver.git import gitsource
class GitDriver(Driver, ConnectionInterface, SourceInterface):
diff --git a/zuul/driver/github/__init__.py b/zuul/driver/github/__init__.py
new file mode 100644
index 0000000..e59dc58
--- /dev/null
+++ b/zuul/driver/github/__init__.py
@@ -0,0 +1,43 @@
+# Copyright 2017 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from zuul.driver import Driver, ConnectionInterface, TriggerInterface
+from zuul.driver import SourceInterface
+from zuul.driver.github import githubconnection
+from zuul.driver.github import githubtrigger
+from zuul.driver.github import githubsource
+from zuul.driver.github import githubreporter
+
+
+class GithubDriver(Driver, ConnectionInterface, TriggerInterface,
+ SourceInterface):
+ name = 'github'
+
+ def getConnection(self, name, config):
+ return githubconnection.GithubConnection(self, name, config)
+
+ def getTrigger(self, connection, config=None):
+ return githubtrigger.GithubTrigger(self, connection, config)
+
+ def getSource(self, connection):
+ return githubsource.GithubSource(self, connection)
+
+ def getReporter(self, connection, config=None):
+ return githubreporter.GithubReporter(self, connection, config)
+
+ def getTriggerSchema(self):
+ return githubtrigger.getSchema()
+
+ def getReporterSchema(self):
+ return githubreporter.getSchema()
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
new file mode 100644
index 0000000..b7fb05d
--- /dev/null
+++ b/zuul/driver/github/githubconnection.py
@@ -0,0 +1,446 @@
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import collections
+import logging
+import hmac
+import hashlib
+import time
+
+import webob
+import webob.dec
+import voluptuous as v
+import github3
+from github3.exceptions import MethodNotAllowed
+
+from zuul.connection import BaseConnection
+from zuul.model import PullRequest, Ref, GithubTriggerEvent
+from zuul.exceptions import MergeFailure
+
+
+class GithubWebhookListener():
+
+ log = logging.getLogger("zuul.GithubWebhookListener")
+
+ def __init__(self, connection):
+ self.connection = connection
+
+ def handle_request(self, path, tenant_name, request):
+ if request.method != 'POST':
+ self.log.debug("Only POST method is allowed.")
+ raise webob.exc.HTTPMethodNotAllowed(
+ 'Only POST method is allowed.')
+
+ self.log.debug("Github Webhook Received.")
+
+ self._validate_signature(request)
+
+ self.__dispatch_event(request)
+
+ def __dispatch_event(self, request):
+ try:
+ event = request.headers['X-Github-Event']
+ self.log.debug("X-Github-Event: " + event)
+ except KeyError:
+ self.log.debug("Request headers missing the X-Github-Event.")
+ raise webob.exc.HTTPBadRequest('Please specify a X-Github-Event '
+ 'header.')
+
+ try:
+ method = getattr(self, '_event_' + event)
+ except AttributeError:
+ message = "Unhandled X-Github-Event: {0}".format(event)
+ self.log.debug(message)
+ raise webob.exc.HTTPBadRequest(message)
+
+ try:
+ event = method(request)
+ except:
+ self.log.exception('Exception when handling event:')
+
+ if event:
+ event.project_hostname = self.connection.canonical_hostname
+ self.log.debug('Scheduling github event: {0}'.format(event.type))
+ self.connection.sched.addEvent(event)
+
+ def _event_push(self, request):
+ body = request.json_body
+ base_repo = body.get('repository')
+
+ event = GithubTriggerEvent()
+ event.trigger_name = 'github'
+ event.project_name = base_repo.get('full_name')
+ event.type = 'push'
+
+ event.ref = body.get('ref')
+ event.oldrev = body.get('before')
+ event.newrev = body.get('after')
+
+ ref_parts = event.ref.split('/') # ie, ['refs', 'heads', 'master']
+
+ if ref_parts[1] == "heads":
+ # necessary for the scheduler to match against particular branches
+ event.branch = ref_parts[2]
+
+ return event
+
+ def _event_pull_request(self, request):
+ body = request.json_body
+ action = body.get('action')
+ pr_body = body.get('pull_request')
+
+ event = self._pull_request_to_event(pr_body)
+ event.account = self._get_sender(body)
+
+ event.type = 'pull_request'
+ if action == 'opened':
+ event.action = 'opened'
+ elif action == 'synchronize':
+ event.action = 'changed'
+ elif action == 'closed':
+ event.action = 'closed'
+ elif action == 'reopened':
+ event.action = 'reopened'
+ elif action == 'labeled':
+ event.action = 'labeled'
+ event.label = body['label']['name']
+ elif action == 'unlabeled':
+ event.action = 'unlabeled'
+ event.label = body['label']['name']
+ else:
+ return None
+
+ return event
+
+ def _event_issue_comment(self, request):
+ """Handles pull request comments"""
+ body = request.json_body
+ action = body.get('action')
+ if action != 'created':
+ return
+ pr_body = self._issue_to_pull_request(body)
+ number = body.get('issue').get('number')
+ project_name = body.get('repository').get('full_name')
+ pr_body = self.connection.getPull(project_name, number)
+ if pr_body is None:
+ return
+
+ event = self._pull_request_to_event(pr_body)
+ event.account = self._get_sender(body)
+ event.comment = body.get('comment').get('body')
+ event.type = 'pull_request'
+ event.action = 'comment'
+ return event
+
+ def _event_pull_request_review(self, request):
+ """Handles pull request reviews"""
+ body = request.json_body
+ pr_body = body.get('pull_request')
+ if pr_body is None:
+ return
+
+ review = body.get('review')
+ if review is None:
+ return
+
+ event = self._pull_request_to_event(pr_body)
+ event.state = review.get('state')
+ event.account = self._get_sender(body)
+ event.type = 'pull_request_review'
+ event.action = body.get('action')
+ return event
+
+ def _issue_to_pull_request(self, body):
+ number = body.get('issue').get('number')
+ project_name = body.get('repository').get('full_name')
+ pr_body = self.connection.getPull(project_name, number)
+ if pr_body is None:
+ self.log.debug('Pull request #%s not found in project %s' %
+ (number, project_name))
+ return pr_body
+
+ def _validate_signature(self, request):
+ secret = self.connection.connection_config.get('webhook_token', None)
+ if secret is None:
+ return True
+
+ body = request.body
+ try:
+ request_signature = request.headers['X-Hub-Signature']
+ except KeyError:
+ raise webob.exc.HTTPUnauthorized(
+ 'Please specify a X-Hub-Signature header with secret.')
+
+ payload_signature = 'sha1=' + hmac.new(secret,
+ body,
+ hashlib.sha1).hexdigest()
+
+ self.log.debug("Payload Signature: {0}".format(str(payload_signature)))
+ self.log.debug("Request Signature: {0}".format(str(request_signature)))
+ if str(payload_signature) != str(request_signature):
+ raise webob.exc.HTTPUnauthorized(
+ 'Request signature does not match calculated payload '
+ 'signature. Check that secret is correct.')
+
+ return True
+
+ def _pull_request_to_event(self, pr_body):
+ event = GithubTriggerEvent()
+ event.trigger_name = 'github'
+
+ base = pr_body.get('base')
+ base_repo = base.get('repo')
+ head = pr_body.get('head')
+
+ event.project_name = base_repo.get('full_name')
+ event.change_number = pr_body.get('number')
+ event.change_url = self.connection.getPullUrl(event.project_name,
+ event.change_number)
+ event.updated_at = pr_body.get('updated_at')
+ event.branch = base.get('ref')
+ event.refspec = "refs/pull/" + str(pr_body.get('number')) + "/head"
+ event.patch_number = head.get('sha')
+
+ event.title = pr_body.get('title')
+
+ return event
+
+ def _get_sender(self, body):
+ login = body.get('sender').get('login')
+ if login:
+ return self.connection.getUser(login)
+
+
+class GithubUser(collections.Mapping):
+ log = logging.getLogger('zuul.GithubUser')
+
+ def __init__(self, github, username):
+ self._github = github
+ self._username = username
+ self._data = None
+
+ def __getitem__(self, key):
+ if self._data is None:
+ self._data = self._init_data()
+ return self._data[key]
+
+ def __iter__(self):
+ return iter(self._data)
+
+ def __len__(self):
+ return len(self._data)
+
+ def _init_data(self):
+ user = self._github.user(self._username)
+ log_rate_limit(self.log, self._github)
+ data = {
+ 'username': user.login,
+ 'name': user.name,
+ 'email': user.email
+ }
+ return data
+
+
+class GithubConnection(BaseConnection):
+ driver_name = 'github'
+ log = logging.getLogger("connection.github")
+ payload_path = 'payload'
+ git_user = 'git'
+
+ def __init__(self, driver, connection_name, connection_config):
+ super(GithubConnection, self).__init__(
+ driver, connection_name, connection_config)
+ self.github = None
+ self._change_cache = {}
+ self.projects = {}
+ self._git_ssh = bool(self.connection_config.get('sshkey', None))
+ self.git_host = self.connection_config.get('git_host', 'github.com')
+ self.canonical_hostname = self.connection_config.get(
+ 'canonical_hostname', self.git_host)
+ self.source = driver.getSource(self)
+
+ def onLoad(self):
+ webhook_listener = GithubWebhookListener(self)
+ self.registerHttpHandler(self.payload_path,
+ webhook_listener.handle_request)
+ self._authenticateGithubAPI()
+
+ def onStop(self):
+ self.unregisterHttpHandler(self.payload_path)
+
+ def _authenticateGithubAPI(self):
+ token = self.connection_config.get('api_token', None)
+ if token is not None:
+ if self.git_host != 'github.com':
+ url = 'https://%s/' % self.git_host
+ self.github = github3.enterprise_login(token=token, url=url)
+ else:
+ self.github = github3.login(token=token)
+ self.log.info("Github API Authentication successful.")
+ else:
+ self.github = None
+ self.log.info(
+ "No Github credentials found in zuul configuration, cannot "
+ "authenticate.")
+
+ def maintainCache(self, relevant):
+ for key, change in self._change_cache.items():
+ if change not in relevant:
+ del self._change_cache[key]
+
+ def getChange(self, event):
+ """Get the change representing an event."""
+
+ project = self.source.getProject(event.project_name)
+ if event.change_number:
+ change = PullRequest(event.project_name)
+ change.project = project
+ change.number = event.change_number
+ change.refspec = event.refspec
+ change.branch = event.branch
+ change.url = event.change_url
+ change.updated_at = self._ghTimestampToDate(event.updated_at)
+ change.patchset = event.patch_number
+ change.files = self.getPullFileNames(project, change.number)
+ change.title = event.title
+ change.source_event = event
+ elif event.ref:
+ change = Ref(project)
+ change.ref = event.ref
+ change.oldrev = event.oldrev
+ change.newrev = event.newrev
+ change.url = self.getGitwebUrl(project, sha=event.newrev)
+ change.source_event = event
+ else:
+ change = Ref(project)
+ return change
+
+ def getGitUrl(self, project):
+ if self._git_ssh:
+ url = 'ssh://%s@%s/%s.git' % \
+ (self.git_user, self.git_host, project)
+ else:
+ url = 'https://%s/%s' % (self.git_host, project)
+ return url
+
+ def getGitwebUrl(self, project, sha=None):
+ url = 'https://%s/%s' % (self.git_host, project)
+ if sha is not None:
+ url += '/commit/%s' % sha
+ return url
+
+ def getProject(self, name):
+ return self.projects.get(name)
+
+ def addProject(self, project):
+ self.projects[project.name] = project
+
+ def getProjectBranches(self, project):
+ owner, proj = project.name.split('/')
+ repository = self.github.repository(owner, proj)
+ branches = [branch.name for branch in repository.branches()]
+ log_rate_limit(self.log, self.github)
+ return branches
+
+ def getPullUrl(self, project, number):
+ return '%s/pull/%s' % (self.getGitwebUrl(project), number)
+
+ def getPull(self, project_name, number):
+ owner, proj = project_name.split('/')
+ pr = self.github.pull_request(owner, proj, number).as_dict()
+ log_rate_limit(self.log, self.github)
+ return pr
+
+ def canMerge(self, change, allow_needs):
+ # This API call may get a false (null) while GitHub is calculating
+ # if it can merge. The github3.py library will just return that as
+ # false. This could lead to false negatives.
+ # Additionally, this only checks if the PR code could merge
+ # cleanly to the target branch. It does not evaluate any branch
+ # protection merge requirements (such as reviews and status states)
+ # At some point in the future this may be available through the API
+ # or we can fetch the branch protection settings and evaluate within
+ # Zuul whether or not those protections have been met
+ # For now, just send back a True value.
+ return True
+
+ def getPullFileNames(self, project, number):
+ owner, proj = project.name.split('/')
+ filenames = [f.filename for f in
+ self.github.pull_request(owner, proj, number).files()]
+ log_rate_limit(self.log, self.github)
+ return filenames
+
+ def getUser(self, login):
+ return GithubUser(self.github, login)
+
+ def getUserUri(self, login):
+ return 'https://%s/%s' % (self.git_host, login)
+
+ def commentPull(self, project, pr_number, message):
+ owner, proj = project.split('/')
+ repository = self.github.repository(owner, proj)
+ pull_request = repository.issue(pr_number)
+ pull_request.create_comment(message)
+ log_rate_limit(self.log, self.github)
+
+ def mergePull(self, project, pr_number, commit_message='', sha=None):
+ owner, proj = project.split('/')
+ pull_request = self.github.pull_request(owner, proj, pr_number)
+ try:
+ result = pull_request.merge(commit_message=commit_message, sha=sha)
+ except MethodNotAllowed as e:
+ raise MergeFailure('Merge was not successful due to mergeability'
+ ' conflict, original error is %s' % e)
+ log_rate_limit(self.log, self.github)
+ if not result:
+ raise Exception('Pull request was not merged')
+
+ def setCommitStatus(self, project, sha, state, url='', description='',
+ context=''):
+ owner, proj = project.split('/')
+ repository = self.github.repository(owner, proj)
+ repository.create_status(sha, state, url, description, context)
+ log_rate_limit(self.log, self.github)
+
+ def labelPull(self, project, pr_number, label):
+ owner, proj = project.split('/')
+ pull_request = self.github.issue(owner, proj, pr_number)
+ pull_request.add_labels(label)
+ log_rate_limit(self.log, self.github)
+
+ def unlabelPull(self, project, pr_number, label):
+ owner, proj = project.split('/')
+ pull_request = self.github.issue(owner, proj, pr_number)
+ pull_request.remove_label(label)
+ log_rate_limit(self.log, self.github)
+
+ def _ghTimestampToDate(self, timestamp):
+ return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
+
+
+def log_rate_limit(log, github):
+ try:
+ rate_limit = github.rate_limit()
+ remaining = rate_limit['resources']['core']['remaining']
+ reset = rate_limit['resources']['core']['reset']
+ except:
+ return
+ log.debug('GitHub API rate limit remaining: %s reset: %s' %
+ (remaining, reset))
+
+
+def getSchema():
+ github_connection = v.Any(str, v.Schema({}, extra=True))
+ return github_connection
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
new file mode 100644
index 0000000..ffec26a
--- /dev/null
+++ b/zuul/driver/github/githubreporter.py
@@ -0,0 +1,156 @@
+# Copyright 2015 Puppet Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+import time
+
+from zuul.reporter import BaseReporter
+from zuul.exceptions import MergeFailure
+
+
+class GithubReporter(BaseReporter):
+ """Sends off reports to Github."""
+
+ name = 'github'
+ log = logging.getLogger("zuul.GithubReporter")
+
+ def __init__(self, driver, connection, config=None):
+ super(GithubReporter, self).__init__(driver, connection, config)
+ self._commit_status = self.config.get('status', None)
+ self._create_comment = self.config.get('comment', True)
+ self._merge = self.config.get('merge', False)
+ self._labels = self.config.get('label', [])
+ if not isinstance(self._labels, list):
+ self._labels = [self._labels]
+ self._unlabels = self.config.get('unlabel', [])
+ if not isinstance(self._unlabels, list):
+ self._unlabels = [self._unlabels]
+
+ def report(self, source, pipeline, item):
+ """Comment on PR and set commit status."""
+ if self._create_comment:
+ self.addPullComment(pipeline, item)
+ if (self._commit_status is not None and
+ hasattr(item.change, 'patchset') and
+ item.change.patchset is not None):
+ self.setPullStatus(pipeline, item)
+ if (self._merge and
+ hasattr(item.change, 'number')):
+ self.mergePull(item)
+ if self._labels or self._unlabels:
+ self.setLabels(item)
+
+ def addPullComment(self, pipeline, item):
+ message = self._formatItemReport(pipeline, item)
+ project = item.change.project.name
+ pr_number = item.change.number
+ self.log.debug(
+ 'Reporting change %s, params %s, message: %s' %
+ (item.change, self.config, message))
+ self.connection.commentPull(project, pr_number, message)
+
+ def setPullStatus(self, pipeline, item):
+ project = item.change.project.name
+ sha = item.change.patchset
+ context = pipeline.name
+ state = self._commit_status
+ url = ''
+ if self.connection.sched.config.has_option('zuul', 'status_url'):
+ base = self.connection.sched.config.get('zuul', 'status_url')
+ url = '%s/#%s,%s' % (base,
+ item.change.number,
+ item.change.patchset)
+ description = ''
+ if pipeline.description:
+ description = pipeline.description
+
+ self.log.debug(
+ 'Reporting change %s, params %s, status:\n'
+ 'context: %s, state: %s, description: %s, url: %s' %
+ (item.change, self.config, context, state,
+ description, url))
+
+ self.connection.setCommitStatus(
+ project, sha, state, url, description, context)
+
+ def mergePull(self, item):
+ project = item.change.project.name
+ pr_number = item.change.number
+ sha = item.change.patchset
+ self.log.debug('Reporting change %s, params %s, merging via API' %
+ (item.change, self.config))
+ message = self._formatMergeMessage(item.change)
+ try:
+ self.connection.mergePull(project, pr_number, message, sha)
+ except MergeFailure:
+ time.sleep(2)
+ self.log.debug('Trying to merge change %s again...' % item.change)
+ self.connection.mergePull(project, pr_number, message, sha)
+ item.change.is_merged = True
+
+ def setLabels(self, item):
+ project = item.change.project.name
+ pr_number = item.change.number
+ if self._labels:
+ self.log.debug('Reporting change %s, params %s, labels:\n%s' %
+ (item.change, self.config, self._labels))
+ for label in self._labels:
+ self.connection.labelPull(project, pr_number, label)
+ if self._unlabels:
+ self.log.debug('Reporting change %s, params %s, unlabels:\n%s' %
+ (item.change, self.config, self._unlabels))
+ for label in self._unlabels:
+ self.connection.unlabelPull(project, pr_number, label)
+
+ def _formatMergeMessage(self, change):
+ message = ''
+
+ if change.title:
+ message += change.title
+
+ account = change.source_event.account
+ if not account:
+ return message
+
+ username = account['username']
+ name = account['name']
+ email = account['email']
+ message += '\n\nReviewed-by: '
+
+ if name:
+ message += name
+ if email:
+ if name:
+ message += ' '
+ message += '<' + email + '>'
+ if name or email:
+ message += '\n '
+ message += self.connection.getUserUri(username)
+
+ return message
+
+
+def getSchema():
+ def toList(x):
+ return v.Any([x], x)
+
+ github_reporter = v.Schema({
+ 'status': v.Any('pending', 'success', 'failure'),
+ 'comment': bool,
+ 'merge': bool,
+ 'label': toList(str),
+ 'unlabel': toList(str)
+ })
+ return github_reporter
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
new file mode 100644
index 0000000..312ee87
--- /dev/null
+++ b/zuul/driver/github/githubsource.py
@@ -0,0 +1,92 @@
+# Copyright 2014 Puppet Labs Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import time
+
+from zuul.source import BaseSource
+from zuul.model import Project
+
+
+class GithubSource(BaseSource):
+ name = 'github'
+ log = logging.getLogger("zuul.source.GithubSource")
+
+ def __init__(self, driver, connection, config=None):
+ hostname = connection.canonical_hostname
+ super(GithubSource, self).__init__(driver, connection,
+ hostname, config)
+
+ def getRefSha(self, project, ref):
+ """Return a sha for a given project ref."""
+ raise NotImplementedError()
+
+ def waitForRefSha(self, project, ref, old_sha=''):
+ """Block until a ref shows up in a given project."""
+ raise NotImplementedError()
+
+ def isMerged(self, change, head=None):
+ """Determine if change is merged."""
+ if not change.number:
+ # Not a pull request, considering merged.
+ return True
+ return change.is_merged
+
+ def canMerge(self, change, allow_needs):
+ """Determine if change can merge."""
+
+ if not change.number:
+ # Not a pull request, considering merged.
+ return True
+ return self.connection.canMerge(change, allow_needs)
+
+ def postConfig(self):
+ """Called after configuration has been processed."""
+ pass
+
+ def getChange(self, event):
+ return self.connection.getChange(event)
+
+ def getProject(self, name):
+ p = self.connection.getProject(name)
+ if not p:
+ p = Project(name, self)
+ self.connection.addProject(p)
+ return p
+
+ def getProjectBranches(self, project):
+ return self.connection.getProjectBranches(project)
+
+ def getProjectOpenChanges(self, project):
+ """Get the open changes for a project."""
+ raise NotImplementedError()
+
+ def updateChange(self, change, history=None):
+ """Update information for a change."""
+ raise NotImplementedError()
+
+ def getGitUrl(self, project):
+ """Get the git url for a project."""
+ return self.connection.getGitUrl(project)
+
+ def getGitwebUrl(self, project, sha=None):
+ """Get the git-web url for a project."""
+ return self.connection.getGitwebUrl(project, sha)
+
+ def getPullFiles(self, project, number):
+ """Get filenames of the pull request"""
+ return self.connection.getPullFileNames(project, number)
+
+ def _ghTimestampToDate(self, timestamp):
+ return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
diff --git a/zuul/driver/github/githubtrigger.py b/zuul/driver/github/githubtrigger.py
new file mode 100644
index 0000000..b9c1026
--- /dev/null
+++ b/zuul/driver/github/githubtrigger.py
@@ -0,0 +1,72 @@
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+from zuul.model import EventFilter
+from zuul.trigger import BaseTrigger
+
+
+class GithubTrigger(BaseTrigger):
+ name = 'github'
+ log = logging.getLogger("zuul.trigger.GithubTrigger")
+
+ def getEventFilters(self, trigger_config):
+ def toList(item):
+ if not item:
+ return []
+ if isinstance(item, list):
+ return item
+ return [item]
+
+ efilters = []
+ for trigger in toList(trigger_config):
+ f = EventFilter(
+ trigger=self,
+ types=toList(trigger['event']),
+ actions=toList(trigger.get('action')),
+ branches=toList(trigger.get('branch')),
+ refs=toList(trigger.get('ref')),
+ comments=toList(trigger.get('comment')),
+ labels=toList(trigger.get('label')),
+ unlabels=toList(trigger.get('unlabel')),
+ states=toList(trigger.get('state'))
+ )
+ efilters.append(f)
+
+ return efilters
+
+ def onPullRequest(self, payload):
+ pass
+
+
+def getSchema():
+ def toList(x):
+ return v.Any([x], x)
+
+ github_trigger = {
+ v.Required('event'):
+ toList(v.Any('pull_request',
+ 'pull_request_review',
+ 'push')),
+ 'action': toList(str),
+ 'branch': toList(str),
+ 'ref': toList(str),
+ 'comment': toList(str),
+ 'label': toList(str),
+ 'unlabel': toList(str),
+ 'state': toList(str),
+ }
+
+ return github_trigger
diff --git a/zuul/driver/smtp/__init__.py b/zuul/driver/smtp/__init__.py
index 0745644..b914c81 100644
--- a/zuul/driver/smtp/__init__.py
+++ b/zuul/driver/smtp/__init__.py
@@ -13,8 +13,8 @@
# under the License.
from zuul.driver import Driver, ConnectionInterface, ReporterInterface
-import smtpconnection
-import smtpreporter
+from zuul.driver.smtp import smtpconnection
+from zuul.driver.smtp import smtpreporter
class SMTPDriver(Driver, ConnectionInterface, ReporterInterface):
diff --git a/zuul/driver/sql/__init__.py b/zuul/driver/sql/__init__.py
index a5f8923..3748e47 100644
--- a/zuul/driver/sql/__init__.py
+++ b/zuul/driver/sql/__init__.py
@@ -13,8 +13,8 @@
# under the License.
from zuul.driver import Driver, ConnectionInterface, ReporterInterface
-import sqlconnection
-import sqlreporter
+from zuul.driver.sql import sqlconnection
+from zuul.driver.sql import sqlreporter
class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
diff --git a/zuul/driver/timer/__init__.py b/zuul/driver/timer/__init__.py
index 115e6af..bca91a1 100644
--- a/zuul/driver/timer/__init__.py
+++ b/zuul/driver/timer/__init__.py
@@ -21,7 +21,7 @@
from zuul.driver import Driver, TriggerInterface
from zuul.model import TriggerEvent
-import timertrigger
+from zuul.driver.timer import timertrigger
class TimerDriver(Driver, TriggerInterface):
diff --git a/zuul/driver/zuul/__init__.py b/zuul/driver/zuul/__init__.py
index 8c9d795..4c3be3d 100644
--- a/zuul/driver/zuul/__init__.py
+++ b/zuul/driver/zuul/__init__.py
@@ -17,7 +17,7 @@
from zuul.driver import Driver, TriggerInterface
from zuul.model import TriggerEvent
-import zuultrigger
+from zuul.driver.zuul import zuultrigger
PARENT_CHANGE_ENQUEUED = 'parent-change-enqueued'
PROJECT_CHANGE_MERGED = 'project-change-merged'
diff --git a/zuul/executor/ansiblelaunchserver.py b/zuul/executor/ansiblelaunchserver.py
index 0202bdd..18762b2 100644
--- a/zuul/executor/ansiblelaunchserver.py
+++ b/zuul/executor/ansiblelaunchserver.py
@@ -59,7 +59,7 @@
return bool(x)
-class LaunchGearWorker(gear.Worker):
+class LaunchGearWorker(gear.TextWorker):
def __init__(self, *args, **kw):
self.__launch_server = kw.pop('launch_server')
super(LaunchGearWorker, self).__init__(*args, **kw)
@@ -71,7 +71,7 @@
return super(LaunchGearWorker, self).handleNoop(packet)
-class NodeGearWorker(gear.Worker):
+class NodeGearWorker(gear.TextWorker):
MASS_DO = 101
def sendMassDo(self, functions):
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 9f234e9..e1eed2d 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -308,8 +308,8 @@
self.sched.onBuildCompleted(build, 'SUCCESS')
return build
- gearman_job = gear.Job('executor:execute', json.dumps(params),
- unique=uuid)
+ gearman_job = gear.TextJob('executor:execute', json.dumps(params),
+ unique=uuid)
build.__gearman_job = gearman_job
build.__gearman_manager = None
self.builds[uuid] = build
@@ -438,7 +438,7 @@
job.connection.sendAdminRequest(req, timeout=300)
self.log.debug("Response to cancel build %s request: %s" %
(build, req.response.strip()))
- if req.response.startswith("OK"):
+ if req.response.startswith(b"OK"):
try:
del self.builds[job.unique]
except:
@@ -452,8 +452,8 @@
(build,))
stop_uuid = str(uuid4().hex)
data = dict(uuid=build.__gearman_job.unique)
- stop_job = gear.Job("executor:stop:%s" % build.__gearman_manager,
- json.dumps(data), unique=stop_uuid)
+ stop_job = gear.TextJob("executor:stop:%s" % build.__gearman_manager,
+ json.dumps(data), unique=stop_uuid)
self.meta_jobs[stop_uuid] = stop_job
self.log.debug("Submitting stop job: %s", stop_job)
self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 4801de2..83c3a1c 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -219,6 +219,16 @@
self.condition.release()
+def _copy_ansible_files(python_module, target_dir):
+ library_path = os.path.dirname(os.path.abspath(python_module.__file__))
+ for fn in os.listdir(library_path):
+ full_path = os.path.join(library_path, fn)
+ if os.path.isdir(full_path):
+ shutil.copytree(full_path, os.path.join(target_dir, fn))
+ else:
+ shutil.copy(os.path.join(library_path, fn), target_dir)
+
+
class ExecutorServer(object):
log = logging.getLogger("zuul.ExecutorServer")
@@ -286,25 +296,10 @@
if not os.path.exists(self.lookup_dir):
os.makedirs(self.lookup_dir)
- library_path = os.path.dirname(os.path.abspath(
- zuul.ansible.library.__file__))
- for fn in os.listdir(library_path):
- shutil.copy(os.path.join(library_path, fn), self.library_dir)
-
- action_path = os.path.dirname(os.path.abspath(
- zuul.ansible.action.__file__))
- for fn in os.listdir(action_path):
- shutil.copy(os.path.join(action_path, fn), self.action_dir)
-
- callback_path = os.path.dirname(os.path.abspath(
- zuul.ansible.callback.__file__))
- for fn in os.listdir(callback_path):
- shutil.copy(os.path.join(callback_path, fn), self.callback_dir)
-
- lookup_path = os.path.dirname(os.path.abspath(
- zuul.ansible.lookup.__file__))
- for fn in os.listdir(lookup_path):
- shutil.copy(os.path.join(lookup_path, fn), self.lookup_dir)
+ _copy_ansible_files(zuul.ansible.library, self.library_dir)
+ _copy_ansible_files(zuul.ansible.action, self.action_dir)
+ _copy_ansible_files(zuul.ansible.callback, self.callback_dir)
+ _copy_ansible_files(zuul.ansible.lookup, self.lookup_dir)
self.job_workers = {}
@@ -320,7 +315,7 @@
port = self.config.get('gearman', 'port')
else:
port = 4730
- self.worker = gear.Worker('Zuul Executor Server')
+ self.worker = gear.TextWorker('Zuul Executor Server')
self.worker.addServer(server, port)
self.log.debug("Waiting for server")
self.worker.waitForServer()
@@ -354,7 +349,7 @@
self.command_socket.stop()
self.update_queue.put(None)
- for job_worker in self.job_workers.values():
+ for job_worker in list(self.job_workers.values()):
try:
job_worker.stop()
except Exception:
@@ -390,7 +385,7 @@
def runCommand(self):
while self._command_running:
try:
- command = self.command_socket.get()
+ command = self.command_socket.get().decode('utf8')
if command != '_stop':
self.command_map[command]()
except Exception:
@@ -445,7 +440,8 @@
job.sendWorkFail()
except Exception:
self.log.exception("Exception while running job")
- job.sendWorkException(traceback.format_exc())
+ job.sendWorkException(
+ traceback.format_exc().encode('utf8'))
except gear.InterruptedError:
pass
except Exception:
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index bec8ebe..3070be6 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -102,7 +102,8 @@
new_repo = git.Repo.clone_from(git_cache, dest)
self.log.info("Updating origin remote in repo %s to %s",
project, git_upstream)
- new_repo.remotes.origin.config_writer.set('url', git_upstream)
+ new_repo.remotes.origin.config_writer.set('url',
+ git_upstream).release()
else:
self.log.info("Creating repo %s from upstream %s",
project, git_upstream)
diff --git a/zuul/lib/commandsocket.py b/zuul/lib/commandsocket.py
index 1b7fed9..ae62204 100644
--- a/zuul/lib/commandsocket.py
+++ b/zuul/lib/commandsocket.py
@@ -18,7 +18,7 @@
import os
import socket
import threading
-import Queue
+from six.moves import queue
class CommandSocket(object):
@@ -27,7 +27,7 @@
def __init__(self, path):
self.running = False
self.path = path
- self.queue = Queue.Queue()
+ self.queue = queue.Queue()
def start(self):
self.running = True
@@ -46,14 +46,14 @@
self.running = False
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(self.path)
- s.sendall('_stop\n')
+ s.sendall(b'_stop\n')
# The command '_stop' will be ignored by our listener, so
# directly inject it into the queue so that consumers of this
# class which are waiting in .get() are awakened. They can
# either handle '_stop' or just ignore the unknown command and
# then check to see if they should continue to run before
# re-entering their loop.
- self.queue.put('_stop')
+ self.queue.put(b'_stop')
self.socket_thread.join()
def _socketListener(self):
@@ -61,10 +61,10 @@
try:
s, addr = self.socket.accept()
self.log.debug("Accepted socket connection %s" % (s,))
- buf = ''
+ buf = b''
while True:
buf += s.recv(1)
- if buf[-1] == '\n':
+ if buf[-1:] == b'\n':
break
buf = buf.strip()
self.log.debug("Received %s from socket" % (buf,))
@@ -72,7 +72,7 @@
# Because we use '_stop' internally to wake up a
# waiting thread, don't allow it to actually be
# injected externally.
- if buf != '_stop':
+ if buf != b'_stop':
self.queue.put(buf)
except Exception:
self.log.exception("Exception in socket handler")
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 403aca6..720299a 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -18,10 +18,12 @@
import zuul.driver.zuul
import zuul.driver.gerrit
import zuul.driver.git
+import zuul.driver.github
import zuul.driver.smtp
import zuul.driver.timer
import zuul.driver.sql
from zuul.connection import BaseConnection
+from zuul.driver import SourceInterface
class DefaultConnection(BaseConnection):
@@ -40,6 +42,7 @@
self.registerDriver(zuul.driver.zuul.ZuulDriver())
self.registerDriver(zuul.driver.gerrit.GerritDriver())
self.registerDriver(zuul.driver.git.GitDriver())
+ self.registerDriver(zuul.driver.github.GithubDriver())
self.registerDriver(zuul.driver.smtp.SMTPDriver())
self.registerDriver(zuul.driver.timer.TimerDriver())
self.registerDriver(zuul.driver.sql.SQLDriver())
@@ -76,7 +79,7 @@
for driver in self.drivers.values():
driver.stop()
- def configure(self, config):
+ def configure(self, config, source_only=False):
# Register connections from the config
connections = {}
@@ -98,6 +101,13 @@
% (con_config['driver'], con_name))
driver = self.drivers[con_driver]
+
+ # The merger and the reporter only needs source driver.
+ # This makes sure Reporter like the SQLDriver are only created by
+ # the scheduler process
+ if source_only and not issubclass(driver, SourceInterface):
+ continue
+
connection = driver.getConnection(con_name, con_config)
connections[con_name] = connection
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index bdfde48..5b32e5b 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -426,7 +426,7 @@
build.result = 'CANCELED'
canceled = True
canceled_jobs.add(build.job.name)
- for jobname, nodeset in old_build_set.nodesets.items()[:]:
+ for jobname, nodeset in list(old_build_set.nodesets.items()):
if jobname in canceled_jobs:
continue
self.sched.nodepool.returnNodeSet(nodeset)
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 642bc1b..e164195 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -56,7 +56,7 @@
self.__merge_client.onBuildCompleted(job)
-class MergeJob(gear.Job):
+class MergeJob(gear.TextJob):
def __init__(self, *args, **kw):
super(MergeJob, self).__init__(*args, **kw)
self.__event = threading.Event()
@@ -113,12 +113,6 @@
files=files)
self.submitJob('merger:merge', data, build_set, precedence)
- def updateRepo(self, connection_name, project_name, build_set,
- precedence=zuul.model.PRECEDENCE_NORMAL):
- data = dict(connection=connection_name,
- project=project_name)
- self.submitJob('merger:update', data, build_set, precedence)
-
def getFiles(self, connection_name, project_name, branch, files,
precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(connection=connection_name,
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 75c51af..714d643 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -64,13 +64,12 @@
self.local_path))
git.Repo.clone_from(self.remote_url, self.local_path)
repo = git.Repo(self.local_path)
- if self.email:
- repo.config_writer().set_value('user', 'email',
- self.email)
- if self.username:
- repo.config_writer().set_value('user', 'name',
- self.username)
- repo.config_writer().write()
+ with repo.config_writer() as config_writer:
+ if self.email:
+ config_writer.set_value('user', 'email', self.email)
+ if self.username:
+ config_writer.set_value('user', 'name', self.username)
+ config_writer.write()
self._initialized = True
def isInitialized(self):
@@ -200,7 +199,7 @@
tree = repo.commit(commit).tree
for fn in files:
if fn in tree:
- ret[fn] = tree[fn].data_stream.read()
+ ret[fn] = tree[fn].data_stream.read().decode('utf8')
else:
ret[fn] = None
return ret
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 39551c9..c09d7ba 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -54,7 +54,7 @@
port = self.config.get('gearman', 'port')
else:
port = 4730
- self.worker = gear.Worker('Zuul Merger')
+ self.worker = gear.TextWorker('Zuul Merger')
self.worker.addServer(server, port)
self.log.debug("Waiting for server")
self.worker.waitForServer()
@@ -67,7 +67,6 @@
def register(self):
self.worker.registerFunction("merger:merge")
- self.worker.registerFunction("merger:update")
self.worker.registerFunction("merger:cat")
def stop(self):
@@ -88,9 +87,6 @@
if job.name == 'merger:merge':
self.log.debug("Got merge job: %s" % job.unique)
self.merge(job)
- elif job.name == 'merger:update':
- self.log.debug("Got update job: %s" % job.unique)
- self.update(job)
elif job.name == 'merger:cat':
self.log.debug("Got cat job: %s" % job.unique)
self.cat(job)
@@ -119,13 +115,6 @@
result['commit'] = ret
job.sendWorkComplete(json.dumps(result))
- def update(self, job):
- args = json.loads(job.arguments)
- self.merger.updateRepo(args['connection'], args['project'])
- result = dict(updated=True,
- zuul_url=self.zuul_url)
- job.sendWorkComplete(json.dumps(result))
-
def cat(self, job):
args = json.loads(job.arguments)
self.merger.updateRepo(args['connection'], args['project'])
diff --git a/zuul/model.py b/zuul/model.py
index 73dec39..7f6223b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -467,7 +467,7 @@
self.nodes[node.name] = node
def getNodes(self):
- return self.nodes.values()
+ return list(self.nodes.values())
def __repr__(self):
if self.name:
@@ -490,9 +490,10 @@
self.stat = None
self.uid = uuid4().hex
self.id = None
- # Zuul internal failure flag (not stored in ZK so it's not
+ # Zuul internal flags (not stored in ZK so they are not
# overwritten).
self.failed = False
+ self.canceled = False
@property
def fulfilled(self):
@@ -681,6 +682,8 @@
def __repr__(self):
return '<ZuulRole %s %s>' % (self.project_name, self.target_name)
+ __hash__ = object.__hash__
+
def __eq__(self, other):
if not isinstance(other, ZuulRole):
return False
@@ -808,6 +811,8 @@
return False
return True
+ __hash__ = object.__hash__
+
def __str__(self):
return self.name
@@ -985,7 +990,7 @@
raise
def getJobs(self):
- return self.jobs.values() # Report in the order of the layout config
+ return list(self.jobs.values()) # Report in the order of layout cfg
def _getDirectDependentJobs(self, parent_job):
ret = set()
@@ -1202,7 +1207,7 @@
return self.builds.get(job_name)
def getBuilds(self):
- keys = self.builds.keys()
+ keys = list(self.builds.keys())
keys.sort()
return [self.builds.get(x) for x in keys]
@@ -1231,7 +1236,7 @@
del self.node_requests[job_name]
def getTries(self, job_name):
- return self.tries.get(job_name)
+ return self.tries.get(job_name, 0)
def getMergeMode(self):
if self.layout:
@@ -1526,11 +1531,11 @@
except KeyError as e:
self.log.error("Error while formatting url for job %s: unknown "
"key %s in pattern %s"
- % (job, e.message, url_pattern))
+ % (job, e.args[0], url_pattern))
except AttributeError as e:
self.log.error("Error while formatting url for job %s: unknown "
"attribute %s in pattern %s"
- % (job, e.message, url_pattern))
+ % (job, e.args[0], url_pattern))
except Exception:
self.log.exception("Error while formatting url for job %s with "
"pattern %s:" % (job, url_pattern))
@@ -1810,6 +1815,8 @@
self.status = None
self.owner = None
+ self.source_event = None
+
def _id(self):
return '%s,%s' % (self.number, self.patchset)
@@ -1819,7 +1826,7 @@
def getBasePath(self):
if hasattr(self, 'refspec'):
return "%s/%s/%s" % (
- self.number[-2:], self.number, self.patchset)
+ str(self.number)[-2:], self.number, self.patchset)
return super(Change, self).getBasePath()
def equals(self, other):
@@ -1856,6 +1863,21 @@
patchset=self.patchset)
+class PullRequest(Change):
+ def __init__(self, project):
+ super(PullRequest, self).__init__(project)
+ self.updated_at = None
+ self.title = None
+
+ def isUpdateOf(self, other):
+ if (hasattr(other, 'number') and self.number == other.number and
+ hasattr(other, 'patchset') and self.patchset != other.patchset and
+ hasattr(other, 'updated_at') and
+ self.updated_at > other.updated_at):
+ return True
+ return False
+
+
class TriggerEvent(object):
"""Incoming event from an external system."""
def __init__(self):
@@ -1877,6 +1899,9 @@
self.approvals = []
self.branch = None
self.comment = None
+ self.label = None
+ self.unlabel = None
+ self.state = None
# ref-updated
self.ref = None
self.oldrev = None
@@ -1907,6 +1932,29 @@
return ret
+ def isPatchsetCreated(self):
+ return 'patchset-created' == self.type
+
+ def isChangeAbandoned(self):
+ return 'change-abandoned' == self.type
+
+
+class GithubTriggerEvent(TriggerEvent):
+
+ def __init__(self):
+ super(GithubTriggerEvent, self).__init__()
+ self.title = None
+
+ def isPatchsetCreated(self):
+ if self.type == 'pull_request':
+ return self.action in ['opened', 'changed']
+ return False
+
+ def isChangeAbandoned(self):
+ if self.type == 'pull_request':
+ return 'closed' == self.action
+ return False
+
class BaseFilter(object):
"""Base Class for filtering which Changes and Events to process."""
@@ -2005,7 +2053,8 @@
def __init__(self, trigger, types=[], branches=[], refs=[],
event_approvals={}, comments=[], emails=[], usernames=[],
timespecs=[], required_approvals=[], reject_approvals=[],
- pipelines=[], ignore_deletes=True):
+ pipelines=[], actions=[], labels=[], unlabels=[], states=[],
+ ignore_deletes=True):
super(EventFilter, self).__init__(
required_approvals=required_approvals,
reject_approvals=reject_approvals)
@@ -2024,8 +2073,12 @@
self.emails = [re.compile(x) for x in emails]
self.usernames = [re.compile(x) for x in usernames]
self.pipelines = [re.compile(x) for x in pipelines]
+ self.actions = actions
self.event_approvals = event_approvals
self.timespecs = timespecs
+ self.labels = labels
+ self.unlabels = unlabels
+ self.states = states
self.ignore_deletes = ignore_deletes
def __repr__(self):
@@ -2058,6 +2111,14 @@
ret += ' username_filters: %s' % ', '.join(self._usernames)
if self.timespecs:
ret += ' timespecs: %s' % ', '.join(self.timespecs)
+ if self.actions:
+ ret += ' actions: %s' % ', '.join(self.actions)
+ if self.labels:
+ ret += ' labels: %s' % ', '.join(self.labels)
+ if self.unlabels:
+ ret += ' unlabels: %s' % ', '.join(self.unlabels)
+ if self.states:
+ ret += ' states: %s' % ', '.join(self.states)
ret += '>'
return ret
@@ -2154,6 +2215,26 @@
if self.timespecs and not matches_timespec:
return False
+ # actions are ORed
+ matches_action = False
+ for action in self.actions:
+ if (event.action == action):
+ matches_action = True
+ if self.actions and not matches_action:
+ return False
+
+ # labels are ORed
+ if self.labels and event.label not in self.labels:
+ return False
+
+ # unlabels are ORed
+ if self.unlabels and event.unlabel not in self.unlabels:
+ return False
+
+ # states are ORed
+ if self.states and event.state not in self.states:
+ return False
+
return True
@@ -2252,7 +2333,7 @@
raise Exception("Configuration item dictionaries must have "
"a single key (when parsing %s)" %
(conf,))
- key, value = item.items()[0]
+ key, value = list(item.items())[0]
if key == 'tenant':
self.tenants.append(value)
else:
@@ -2310,7 +2391,7 @@
raise Exception("Configuration item dictionaries must have "
"a single key (when parsing %s)" %
(conf,))
- key, value = item.items()[0]
+ key, value = list(item.items())[0]
if key == 'project':
name = value['name']
self.projects.setdefault(name, []).append(value)
@@ -2617,7 +2698,7 @@
if hostname:
project = hostname_dict.get(hostname)
else:
- values = hostname_dict.values()
+ values = list(hostname_dict.values())
if len(values) == 1:
project = values[0]
else:
@@ -2661,7 +2742,7 @@
def load(self):
if not os.path.exists(self.path):
return
- with open(self.path) as f:
+ with open(self.path, 'rb') as f:
data = struct.unpack(self.format, f.read())
version = data[0]
if version != self.version:
@@ -2677,7 +2758,7 @@
data.extend(self.failure_times)
data.extend(self.results)
data = struct.pack(self.format, *data)
- with open(tmpfile, 'w') as f:
+ with open(tmpfile, 'wb') as f:
f.write(data)
os.rename(tmpfile, self.path)
diff --git a/zuul/nodepool.py b/zuul/nodepool.py
index e94b950..8f6489c 100644
--- a/zuul/nodepool.py
+++ b/zuul/nodepool.py
@@ -38,11 +38,11 @@
def cancelRequest(self, request):
self.log.info("Canceling node request %s" % (request,))
if request.uid in self.requests:
+ request.canceled = True
try:
self.sched.zk.deleteNodeRequest(request)
except Exception:
self.log.exception("Error deleting node request:")
- del self.requests[request.uid]
def useNodeSet(self, nodeset):
self.log.info("Setting nodeset %s in use" % (nodeset,))
@@ -98,6 +98,10 @@
if request.uid not in self.requests:
return False
+ if request.canceled:
+ del self.requests[request.uid]
+ return False
+
if request.state in (model.STATE_FULFILLED, model.STATE_FAILED):
self.log.info("Node request %s %s" % (request, request.state))
@@ -119,6 +123,11 @@
self.log.info("Accepting node request %s" % (request,))
+ if request.canceled:
+ self.log.info("Ignoring canceled node request %s" % (request,))
+ # The request was already deleted when it was canceled
+ return
+
locked = False
if request.fulfilled:
# If the request suceeded, try to lock the nodes.
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 5e25e7c..582265d 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -74,7 +74,12 @@
return ret
def _formatItemReportStart(self, pipeline, item, with_jobs=True):
- return pipeline.start_message.format(pipeline=pipeline)
+ status_url = ''
+ if self.connection.sched.config.has_option('zuul', 'status_url'):
+ status_url = self.connection.sched.config.get('zuul',
+ 'status_url')
+ return pipeline.start_message.format(pipeline=pipeline,
+ status_url=status_url)
def _formatItemReportSuccess(self, pipeline, item, with_jobs=True):
msg = pipeline.success_message
diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py
index 9d81520..d980992 100644
--- a/zuul/rpcclient.py
+++ b/zuul/rpcclient.py
@@ -35,9 +35,9 @@
def submitJob(self, name, data):
self.log.debug("Submitting job %s with data %s" % (name, data))
- job = gear.Job(name,
- json.dumps(data),
- unique=str(time.time()))
+ job = gear.TextJob(name,
+ json.dumps(data),
+ unique=str(time.time()))
self.gearman.submitJob(job, timeout=300)
self.log.debug("Waiting for job completion")
diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py
index 105c34b..6508e84 100644
--- a/zuul/rpclistener.py
+++ b/zuul/rpclistener.py
@@ -38,7 +38,7 @@
port = self.config.get('gearman', 'port')
else:
port = 4730
- self.worker = gear.Worker('Zuul RPC Listener')
+ self.worker = gear.TextWorker('Zuul RPC Listener')
self.worker.addServer(server, port)
self.worker.waitForServer()
self.register()
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 53ca4c1..a67973e 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -759,9 +759,9 @@
change.project.unparsed_config = None
self.reconfigureTenant(tenant)
for pipeline in tenant.layout.pipelines.values():
- if event.type == 'patchset-created':
+ if event.isPatchsetCreated():
pipeline.manager.removeOldVersionsOfChange(change)
- elif event.type == 'change-abandoned':
+ elif event.isChangeAbandoned():
pipeline.manager.removeAbandonedChange(change)
if pipeline.manager.eventMatches(event, change):
pipeline.manager.addChange(change)
@@ -871,6 +871,8 @@
build_set = request.build_set
self.nodepool.acceptNodes(request)
+ if request.canceled:
+ return
if build_set is not build_set.item.current_build_set:
self.log.warning("Build set %s is not current" % (build_set,))
diff --git a/zuul/webapp.py b/zuul/webapp.py
index f5a7373..e4feaa0 100644
--- a/zuul/webapp.py
+++ b/zuul/webapp.py
@@ -45,7 +45,7 @@
class WebApp(threading.Thread):
log = logging.getLogger("zuul.WebApp")
- change_path_regexp = '/status/change/(\d+,\d+)$'
+ change_path_regexp = '/status/change/(.*)$'
def __init__(self, scheduler, port=8001, cache_expiry=1,
listen_address='0.0.0.0'):
@@ -128,7 +128,7 @@
def app(self, request):
# Try registered paths without a tenant_name first
path = request.path
- for path_re, handler in self.routes.itervalues():
+ for path_re, handler in self.routes.values():
if path_re.match(path):
return handler(path, '', request)
@@ -138,7 +138,7 @@
# Handle keys
if path.startswith('/keys'):
return self._handle_keys(request, path)
- for path_re, handler in self.routes.itervalues():
+ for path_re, handler in self.routes.values():
if path_re.match(path):
return handler(path, tenant_name, request)
else:
@@ -147,7 +147,8 @@
def status(self, path, tenant_name, request):
def func():
return webob.Response(body=self.cache[tenant_name],
- content_type='application/json')
+ content_type='application/json',
+ charset='utf8')
return self._response_with_status_cache(func, tenant_name)
def change(self, path, tenant_name, request):
@@ -157,7 +158,8 @@
status = self._status_for_change(change_id, tenant_name)
if status:
return webob.Response(body=status,
- content_type='application/json')
+ content_type='application/json',
+ charset='utf8')
else:
raise webob.exc.HTTPNotFound()
return self._response_with_status_cache(func, tenant_name)
diff --git a/zuul/zk.py b/zuul/zk.py
index 5cd7bee..31b85ea 100644
--- a/zuul/zk.py
+++ b/zuul/zk.py
@@ -59,10 +59,10 @@
self._became_lost = False
def _dictToStr(self, data):
- return json.dumps(data)
+ return json.dumps(data).encode('utf8')
def _strToDict(self, data):
- return json.loads(data)
+ return json.loads(data.decode('utf8'))
def _connection_listener(self, state):
'''
@@ -168,7 +168,7 @@
if data:
data = self._strToDict(data)
node_request.updateFromDict(data)
- request_nodes = node_request.nodeset.getNodes()
+ request_nodes = list(node_request.nodeset.getNodes())
for i, nodeid in enumerate(data.get('nodes', [])):
node_path = '%s/%s' % (self.NODE_ROOT, nodeid)
node_data, node_stat = self.client.get(node_path)