Merge "Re-enable test_live_reconfiguration_failed_root" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index 50223fa..0ae5beb 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,49 +1,3 @@
-- job:
- name: base
- pre-run: base/pre
- post-run: base/post
- success-url: http://zuulv3-dev.openstack.org/logs/{build.uuid}/
- failure-url: http://zuulv3-dev.openstack.org/logs/{build.uuid}/
- timeout: 1800
- vars:
- zuul_workspace_root: /home/zuul
- nodes:
- - name: ubuntu-xenial
- image: ubuntu-xenial
-
-- job:
- name: tox
- parent: base
- pre-run: tox/pre
- post-run: tox/post
-
-- job:
- name: tox-cover
- parent: tox
- run: tox/cover
- voting: false
-
-- job:
- name: tox-docs
- parent: tox
- run: tox/docs
-
-- job:
- name: tox-linters
- parent: tox
- run: tox/linters
-
-- job:
- name: tox-py27
- parent: tox
- run: tox/py27
-
-- job:
- name: tox-tarball
- parent: tox
- run: tox/tarball
- post-run: tox/tarball-post
-
- project:
name: openstack-infra/zuul
check:
diff --git a/doc/source/connections.rst b/doc/source/connections.rst
index 614b44a..7b302e9 100644
--- a/doc/source/connections.rst
+++ b/doc/source/connections.rst
@@ -65,6 +65,28 @@
be added to Gerrit. Zuul is very flexible and can take advantage of
those.
+GitHub
+------
+
+Create a connection with GitHub.
+
+**driver=github**
+
+**api_token**
+ API token for accessing GitHub.
+ See `Creating an access token for command-line use
+ <https://help.github.com/articles/creating-an-access-token-for-command-line-use/>`_.
+
+**webhook_token**
+ Optional: Token for validating the webhook event payloads.
+ If not specified, payloads are not validated.
+ See `Securing your webhooks
+ <https://developer.github.com/webhooks/securing/>`_.
+
+**sshkey**
+ Path to SSH key to use when cloning github repositories.
+ ``sshkey=/home/zuul/.ssh/id_rsa``
+
SMTP
----
diff --git a/doc/source/reporters.rst b/doc/source/reporters.rst
index b01c8d1..e3ab947 100644
--- a/doc/source/reporters.rst
+++ b/doc/source/reporters.rst
@@ -28,6 +28,43 @@
A :ref:`connection` that uses the gerrit driver must be supplied to the
trigger.
+GitHub
+------
+
+Zuul reports back to GitHub pull requests via GitHub API.
+On success and failure, it creates a comment containing the build results.
+It also sets the status on start, success and failure. Status name and
+description is taken from the pipeline.
+
+A :ref:`connection` that uses the github driver must be supplied to the
+reporter. It has the following options:
+
+ **status**
+ String value (``pending``, ``success``, ``failure``) that the reporter should
+ set as the commit status on github.
+ ``status: 'success'``
+
+ **comment**
+ Boolean value (``true`` or ``false``) that determines if the reporter should
+ add a comment to the pipeline status to the github pull request. Defaults
+ to ``true``.
+ ``comment: false``
+
+ **merge**
+ Boolean value (``true`` or ``false``) that determines if the reporter should
+ merge the pull reqeust. Defaults to ``false``.
+ ``merge=true``
+
+ **label**
+ List of strings each representing an exact label name which should be added
+ to the pull request by reporter.
+ ``label: 'test successful'``
+
+ **unlabel**
+ List of strings each representing an exact label name which should be removed
+ from the pull request by reporter.
+ ``unlabel: 'test failed'``
+
SMTP
----
diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst
index 263f280..07b18ab 100644
--- a/doc/source/triggers.rst
+++ b/doc/source/triggers.rst
@@ -4,7 +4,7 @@
========
The process of merging a change starts with proposing a change to be
-merged. Primarily, Zuul supports Gerrit as a triggering system.
+merged. Zuul supports Gerrit and GitHub as triggering systems.
Zuul's design is modular, so alternate triggering and reporting
systems can be supported.
@@ -100,6 +100,71 @@
*require-approval* but will fail to enter the pipeline if there is
a matching approval.
+GitHub
+------
+
+Github webhook events can be configured as triggers.
+
+A connection name with the github driver can take multiple events with the
+following options.
+
+ **event**
+ The pull request event from github. A ``pull_request`` event will
+ have associated action(s) to trigger from. The supported actions are:
+
+ *opened* - pull request opened
+
+ *changed* - pull request synchronized
+
+ *closed* - pull request closed
+
+ *reopened* - pull request reopened
+
+ *comment* - comment added on pull request
+
+ *labeled* - label added on pull request
+
+ *unlabeled* - label removed from pull request
+
+ *push* - head reference updated (pushed to branch)
+
+ **branch**
+ The branch associated with the event. Example: ``master``. This
+ field is treated as a regular expression, and multiple branches may
+ be listed. Used for ``pull-request`` events.
+
+ **comment**
+ This is only used for ``pull_request`` ``comment`` events. It accepts a list
+ of regexes that are searched for in the comment string. If any of these
+ regexes matches a portion of the comment string the trigger is matched.
+ ``comment: retrigger`` will match when comments containing 'retrigger'
+ somewhere in the comment text are added to a pull request.
+
+ **label**
+ This is only used for ``labeled`` and ``unlabeled`` actions. It accepts a list
+ of strings each of which matches the label name in the event literally.
+ ``label: recheck`` will match a ``labeled`` action when pull request is
+ labeled with a ``recheck`` label. ``label: 'do not test'`` will match a
+ ``unlabeled`` action when a label with name ``do not test`` is removed from
+ the pull request.
+
+ Additionally a ``push`` event can be configured, with an ``ref`` field. This
+ field is treated as a regular expression and multiple refs may be listed.
+ Github always sends full ref name, eg. ``refs/tags/bar`` and this string is
+ matched against the regexp.
+
+GitHub Configuration
+~~~~~~~~~~~~~~~~~~~~
+
+Configure GitHub `webhook events
+<https://developer.github.com/webhooks/creating/>`_.
+
+Set *Payload URL* to
+``http://<zuul-hostname>/connection/<connection-name>/payload``.
+
+Set *Content Type* to ``application/json``.
+
+Select *Events* you are interested in. See above for the supported events.
Timer
-----
@@ -154,4 +219,4 @@
*reject-approval*
This takes a list of approvals in the same format as
*require-approval* but will fail to enter the pipeline if there is
- a matching approval.
\ No newline at end of file
+ a matching approval.
diff --git a/playbooks/base/post.yaml b/playbooks/base/post.yaml
deleted file mode 100644
index ed3f7b8..0000000
--- a/playbooks/base/post.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-- hosts: all
- tasks:
- - name: Collect console log.
- synchronize:
- dest: "{{ zuul.executor.log_root }}"
- mode: pull
- src: "/tmp/console.log"
-
- - name: Publish logs.
- copy:
- dest: "/opt/zuul-logs/{{ zuul.uuid}}"
- src: "{{ zuul.executor.log_root }}/"
- delegate_to: 127.0.0.1
diff --git a/playbooks/base/pre.yaml b/playbooks/base/pre.yaml
deleted file mode 100644
index 1a2e699..0000000
--- a/playbooks/base/pre.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-- hosts: all
- roles:
- - prepare-workspace
diff --git a/playbooks/base/roles b/playbooks/base/roles
deleted file mode 120000
index 7b9ade8..0000000
--- a/playbooks/base/roles
+++ /dev/null
@@ -1 +0,0 @@
-../roles/
\ No newline at end of file
diff --git a/playbooks/roles/extra-test-setup/tasks/main.yaml b/playbooks/roles/extra-test-setup/tasks/main.yaml
deleted file mode 100644
index da4259e..0000000
--- a/playbooks/roles/extra-test-setup/tasks/main.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- name: Check if projects tools/test-setup.sh exists.
- stat:
- path: "{{ zuul_workspace_root }}/src/{{ zuul.project }}/tools/test-setup.sh"
- register: p
-
-- name: Run tools/test-setup.sh.
- shell: tools/test-setup.sh
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
- when:
- - p.stat.exists
- - p.stat.executable
diff --git a/playbooks/roles/prepare-workspace/tasks/main.yaml b/playbooks/roles/prepare-workspace/tasks/main.yaml
deleted file mode 100644
index 4d42b2d..0000000
--- a/playbooks/roles/prepare-workspace/tasks/main.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-- name: Ensure console.log does not exist.
- file:
- path: /tmp/console.log
- state: absent
-
-- name: Start zuul_console daemon.
- zuul_console:
- path: /tmp/console.log
- port: 19885
-
-- name: Create workspace directory.
- file:
- path: "{{ zuul_workspace_root }}"
- owner: zuul
- group: zuul
- state: directory
-
-- name: Synchronize src repos to workspace directory.
- synchronize:
- dest: "{{ zuul_workspace_root }}"
- src: "{{ zuul.executor.src_root }}"
- no_log: true
diff --git a/playbooks/roles/revoke-sudo/tasks/main.yaml b/playbooks/roles/revoke-sudo/tasks/main.yaml
deleted file mode 100644
index 1c18187..0000000
--- a/playbooks/roles/revoke-sudo/tasks/main.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-- name: Remove sudo access for zuul user.
- become: yes
- file:
- path: /etc/sudoers.d/zuul-sudo
- state: absent
-
-- name: Prove that general sudo access is actually revoked.
- shell: ! sudo -n true
diff --git a/playbooks/roles/run-bindep/tasks/main.yaml b/playbooks/roles/run-bindep/tasks/main.yaml
deleted file mode 100644
index 5a9d33e..0000000
--- a/playbooks/roles/run-bindep/tasks/main.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
----
-- name: Run install-distro-packages.sh
- shell: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-cover/defaults/main.yaml b/playbooks/roles/run-cover/defaults/main.yaml
deleted file mode 100644
index 2e32efe..0000000
--- a/playbooks/roles/run-cover/defaults/main.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-run_cover_envlist: cover
diff --git a/playbooks/roles/run-cover/tasks/main.yaml b/playbooks/roles/run-cover/tasks/main.yaml
deleted file mode 100644
index caed13c..0000000
--- a/playbooks/roles/run-cover/tasks/main.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- name: Execute run-cover.sh.
- shell: "/usr/local/jenkins/slave_scripts/run-cover.sh {{ run_cover_envlist }}"
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-docs/defaults/main.yaml b/playbooks/roles/run-docs/defaults/main.yaml
deleted file mode 100644
index 5855a3d..0000000
--- a/playbooks/roles/run-docs/defaults/main.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-run_docs_envlist: venv
diff --git a/playbooks/roles/run-docs/tasks/main.yaml b/playbooks/roles/run-docs/tasks/main.yaml
deleted file mode 100644
index 2250593..0000000
--- a/playbooks/roles/run-docs/tasks/main.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- name: Execute run-docs.sh.
- shell: "/usr/local/jenkins/slave_scripts/run-docs.sh {{ run_docs_envlist }}"
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-tarball/defaults/main.yaml b/playbooks/roles/run-tarball/defaults/main.yaml
deleted file mode 100644
index 072828a..0000000
--- a/playbooks/roles/run-tarball/defaults/main.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-run_tarball_envlist: venv
diff --git a/playbooks/roles/run-tarball/tasks/main.yaml b/playbooks/roles/run-tarball/tasks/main.yaml
deleted file mode 100644
index e21c4c8..0000000
--- a/playbooks/roles/run-tarball/tasks/main.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- name: Execute run-tarball.sh.
- shell: "/usr/local/jenkins/slave_scripts/run-tarball.sh {{ run_tarball_envlist }}"
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-tox/defaults/main.yaml b/playbooks/roles/run-tox/defaults/main.yaml
deleted file mode 100644
index 9cb1477..0000000
--- a/playbooks/roles/run-tox/defaults/main.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-run_tox_envlist:
diff --git a/playbooks/roles/run-tox/tasks/main.yaml b/playbooks/roles/run-tox/tasks/main.yaml
deleted file mode 100644
index 29a4cc4..0000000
--- a/playbooks/roles/run-tox/tasks/main.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- name: Run tox
- shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {{ run_tox_envlist }}"
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-wheel/defaults/main.yaml b/playbooks/roles/run-wheel/defaults/main.yaml
deleted file mode 100644
index 8645d33..0000000
--- a/playbooks/roles/run-wheel/defaults/main.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-run_wheel_envlist: venv
diff --git a/playbooks/roles/run-wheel/tasks/main.yaml b/playbooks/roles/run-wheel/tasks/main.yaml
deleted file mode 100644
index f5aaf54..0000000
--- a/playbooks/roles/run-wheel/tasks/main.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- name: Execute run-wheel.sh.
- shell: "/usr/local/jenkins/slave_scripts/run-wheel.sh {{ run_wheel_envlist }}"
- args:
- chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/tox/cover.yaml b/playbooks/tox/cover.yaml
deleted file mode 100644
index 642eb4e..0000000
--- a/playbooks/tox/cover.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: all
- roles:
- - extra-test-setup
- - revoke-sudo
- - run-cover
diff --git a/playbooks/tox/docs.yaml b/playbooks/tox/docs.yaml
deleted file mode 100644
index 028e1c5..0000000
--- a/playbooks/tox/docs.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- hosts: all
- roles:
- - revoke-sudo
- - run-docs
diff --git a/playbooks/tox/linters.yaml b/playbooks/tox/linters.yaml
deleted file mode 100644
index d1e7f13..0000000
--- a/playbooks/tox/linters.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-- hosts: all
- vars:
- run_tox_envlist: pep8
- roles:
- - revoke-sudo
- - run-tox
diff --git a/playbooks/tox/post.yaml b/playbooks/tox/post.yaml
deleted file mode 100644
index 3b035f8..0000000
--- a/playbooks/tox/post.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-- hosts: all
- tasks:
- - name: Find tox directories to synchrionize.
- find:
- file_type: directory
- paths: "{{ zuul_workspace_root }}/src/{{ zuul.project }}/.tox"
- # NOTE(pabelanger): The .tox/log folder is empty, ignore it.
- patterns: ^(?!log).*$
- use_regex: yes
- register: result
-
- - name: Collect tox logs.
- synchronize:
- dest: "{{ zuul.executor.log_root }}/tox"
- mode: pull
- src: "{{ item.path }}/log/"
- with_items: "{{ result.files }}"
diff --git a/playbooks/tox/pre.yaml b/playbooks/tox/pre.yaml
deleted file mode 100644
index 0bf9b3c..0000000
--- a/playbooks/tox/pre.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-- hosts: all
- roles:
- - run-bindep
diff --git a/playbooks/tox/py27.yaml b/playbooks/tox/py27.yaml
deleted file mode 100644
index fd45f27..0000000
--- a/playbooks/tox/py27.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-- hosts: all
- vars:
- run_tox_envlist: py27
- roles:
- - extra-test-setup
- - revoke-sudo
- - run-tox
diff --git a/playbooks/tox/roles b/playbooks/tox/roles
deleted file mode 120000
index 7b9ade8..0000000
--- a/playbooks/tox/roles
+++ /dev/null
@@ -1 +0,0 @@
-../roles/
\ No newline at end of file
diff --git a/playbooks/tox/tarball-post.yaml b/playbooks/tox/tarball-post.yaml
deleted file mode 100644
index fb41707..0000000
--- a/playbooks/tox/tarball-post.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-- hosts: all
- tasks:
- - name: Collect tarball artifacts.
- synchronize:
- dest: "{{ zuul.executor.src_root }}/tarballs"
- mode: pull
- src: "{{ zuul_workspace_root }}/src/{{ zuul.project }}/dist/{{ item }}"
- with_items:
- - "*.tar.gz"
- - "*.whl"
diff --git a/playbooks/tox/tarball.yaml b/playbooks/tox/tarball.yaml
deleted file mode 100644
index 4d5a8f6..0000000
--- a/playbooks/tox/tarball.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: all
- roles:
- - revoke-sudo
- - run-tarball
- - run-wheel
diff --git a/requirements.txt b/requirements.txt
index 974b77f..2fe6963 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
pbr>=1.1.0
+Github3.py==1.0.0a2
PyYAML>=3.1.0
Paste
WebOb>=1.2.3
diff --git a/test-requirements.txt b/test-requirements.txt
index 6262a02..735b4dd 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,7 +1,7 @@
hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0
coverage>=3.6
-sphinx>=1.5.1
+sphinx>=1.5.1,<1.6
sphinxcontrib-blockdiag>=1.1.0
fixtures>=0.3.14
python-keystoneclient>=0.4.2
diff --git a/tests/base.py b/tests/base.py
index 2c3f7bb..77c0644 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -58,6 +58,7 @@
import zuul.driver.gerrit.gerritsource as gerritsource
import zuul.driver.gerrit.gerritconnection as gerritconnection
+import zuul.driver.github.githubconnection as githubconnection
import zuul.scheduler
import zuul.webapp
import zuul.rpclistener
@@ -69,6 +70,7 @@
import zuul.merger.server
import zuul.nodepool
import zuul.zk
+from zuul.exceptions import MergeFailure
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'fixtures')
@@ -126,12 +128,12 @@
return decorator
-class ChangeReference(git.Reference):
+class GerritChangeReference(git.Reference):
_common_path_default = "refs/changes"
_points_to_commits_only = True
-class FakeChange(object):
+class FakeGerritChange(object):
categories = {'approved': ('Approved', -1, 1),
'code-review': ('Code-Review', -2, 2),
'verified': ('Verified', -2, 2)}
@@ -139,6 +141,7 @@
def __init__(self, gerrit, number, project, branch, subject,
status='NEW', upstream_root=None, files={}):
self.gerrit = gerrit
+ self.source = gerrit
self.reported = 0
self.queried = 0
self.patchsets = []
@@ -178,9 +181,9 @@
def addFakeChangeToRepo(self, msg, files, large):
path = os.path.join(self.upstream_root, self.project)
repo = git.Repo(path)
- ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
- self.latest_patchset),
- 'refs/tags/init')
+ ref = GerritChangeReference.create(
+ repo, '1/%s/%s' % (self.number, self.latest_patchset),
+ 'refs/tags/init')
repo.head.reference = ref
zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
@@ -469,9 +472,9 @@
files=None):
"""Add a change to the fake Gerrit."""
self.change_number += 1
- c = FakeChange(self, self.change_number, project, branch, subject,
- upstream_root=self.upstream_root,
- status=status, files=files)
+ c = FakeGerritChange(self, self.change_number, project, branch,
+ subject, upstream_root=self.upstream_root,
+ status=status, files=files)
self.changes[self.change_number] = c
return c
@@ -536,6 +539,325 @@
return os.path.join(self.upstream_root, project.name)
+class GithubChangeReference(git.Reference):
+ _common_path_default = "refs/pull"
+ _points_to_commits_only = True
+
+
+class FakeGithubPullRequest(object):
+
+ def __init__(self, github, number, project, branch,
+ subject, upstream_root, number_of_commits=1):
+ """Creates a new PR with several commits.
+ Sends an event about opened PR."""
+ self.github = github
+ self.source = github
+ self.number = number
+ self.project = project
+ self.branch = branch
+ self.subject = subject
+ self.number_of_commits = 0
+ self.upstream_root = upstream_root
+ self.comments = []
+ self.labels = []
+ self.statuses = {}
+ self.updated_at = None
+ self.head_sha = None
+ self.is_merged = False
+ self._createPRRef()
+ self._addCommitToRepo()
+ self._updateTimeStamp()
+
+ def addCommit(self):
+ """Adds a commit on top of the actual PR head."""
+ self._addCommitToRepo()
+ self._updateTimeStamp()
+ self._clearStatuses()
+
+ def forcePush(self):
+ """Clears actual commits and add a commit on top of the base."""
+ self._addCommitToRepo(reset=True)
+ self._updateTimeStamp()
+ self._clearStatuses()
+
+ def getPullRequestOpenedEvent(self):
+ return self._getPullRequestEvent('opened')
+
+ def getPullRequestSynchronizeEvent(self):
+ return self._getPullRequestEvent('synchronize')
+
+ def getPullRequestReopenedEvent(self):
+ return self._getPullRequestEvent('reopened')
+
+ def getPullRequestClosedEvent(self):
+ return self._getPullRequestEvent('closed')
+
+ def addComment(self, message):
+ self.comments.append(message)
+ self._updateTimeStamp()
+
+ def getCommentAddedEvent(self, text):
+ name = 'issue_comment'
+ data = {
+ 'action': 'created',
+ 'issue': {
+ 'number': self.number
+ },
+ 'comment': {
+ 'body': text
+ },
+ 'repository': {
+ 'full_name': self.project
+ }
+ }
+ return (name, data)
+
+ def addLabel(self, name):
+ if name not in self.labels:
+ self.labels.append(name)
+ self._updateTimeStamp()
+ return self._getLabelEvent(name)
+
+ def removeLabel(self, name):
+ if name in self.labels:
+ self.labels.remove(name)
+ self._updateTimeStamp()
+ return self._getUnlabelEvent(name)
+
+ def _getLabelEvent(self, label):
+ name = 'pull_request'
+ data = {
+ 'action': 'labeled',
+ 'pull_request': {
+ 'number': self.number,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ },
+ 'label': {
+ 'name': label
+ }
+ }
+ return (name, data)
+
+ def _getUnlabelEvent(self, label):
+ name = 'pull_request'
+ data = {
+ 'action': 'unlabeled',
+ 'pull_request': {
+ 'number': self.number,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ },
+ 'label': {
+ 'name': label
+ }
+ }
+ return (name, data)
+
+ def _getRepo(self):
+ repo_path = os.path.join(self.upstream_root, self.project)
+ return git.Repo(repo_path)
+
+ def _createPRRef(self):
+ repo = self._getRepo()
+ GithubChangeReference.create(
+ repo, self._getPRReference(), 'refs/tags/init')
+
+ def _addCommitToRepo(self, reset=False):
+ repo = self._getRepo()
+ ref = repo.references[self._getPRReference()]
+ if reset:
+ self.number_of_commits = 0
+ ref.set_object('refs/tags/init')
+ self.number_of_commits += 1
+ repo.head.reference = ref
+ zuul.merger.merger.reset_repo_to_head(repo)
+ repo.git.clean('-x', '-f', '-d')
+
+ fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
+ msg = self.subject + '-' + str(self.number_of_commits)
+ fn = os.path.join(repo.working_dir, fn)
+ f = open(fn, 'w')
+ with open(fn, 'w') as f:
+ f.write("test %s %s\n" %
+ (self.branch, self.number))
+ repo.index.add([fn])
+
+ self.head_sha = repo.index.commit(msg).hexsha
+ repo.head.reference = 'master'
+ zuul.merger.merger.reset_repo_to_head(repo)
+ repo.git.clean('-x', '-f', '-d')
+ repo.heads['master'].checkout()
+
+ def _updateTimeStamp(self):
+ self.updated_at = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
+
+ def getPRHeadSha(self):
+ repo = self._getRepo()
+ return repo.references[self._getPRReference()].commit.hexsha
+
+ def setStatus(self, state, url, description, context):
+ self.statuses[context] = {
+ 'state': state,
+ 'url': url,
+ 'description': description
+ }
+
+ def _clearStatuses(self):
+ self.statuses = {}
+
+ def _getPRReference(self):
+ return '%s/head' % self.number
+
+ def _getPullRequestEvent(self, action):
+ name = 'pull_request'
+ data = {
+ 'action': action,
+ 'number': self.number,
+ 'pull_request': {
+ 'number': self.number,
+ 'updated_at': self.updated_at,
+ 'base': {
+ 'ref': self.branch,
+ 'repo': {
+ 'full_name': self.project
+ }
+ },
+ 'head': {
+ 'sha': self.head_sha
+ }
+ }
+ }
+ return (name, data)
+
+
+class FakeGithubConnection(githubconnection.GithubConnection):
+ log = logging.getLogger("zuul.test.FakeGithubConnection")
+
+ def __init__(self, driver, connection_name, connection_config,
+ upstream_root=None):
+ super(FakeGithubConnection, self).__init__(driver, connection_name,
+ connection_config)
+ self.connection_name = connection_name
+ self.pr_number = 0
+ self.pull_requests = []
+ self.upstream_root = upstream_root
+ self.merge_failure = False
+ self.merge_not_allowed_count = 0
+
+ def openFakePullRequest(self, project, branch, subject):
+ self.pr_number += 1
+ pull_request = FakeGithubPullRequest(
+ self, self.pr_number, project, branch, subject, self.upstream_root)
+ self.pull_requests.append(pull_request)
+ return pull_request
+
+ def getPushEvent(self, project, ref, old_rev=None, new_rev=None):
+ if not old_rev:
+ old_rev = '00000000000000000000000000000000'
+ if not new_rev:
+ new_rev = random_sha1()
+ name = 'push'
+ data = {
+ 'ref': ref,
+ 'before': old_rev,
+ 'after': new_rev,
+ 'repository': {
+ 'full_name': project
+ }
+ }
+ return (name, data)
+
+ def emitEvent(self, event):
+ """Emulates sending the GitHub webhook event to the connection."""
+ port = self.webapp.server.socket.getsockname()[1]
+ name, data = event
+ payload = json.dumps(data)
+ headers = {'X-Github-Event': name}
+ req = urllib.request.Request(
+ 'http://localhost:%s/connection/%s/payload'
+ % (port, self.connection_name),
+ data=payload, headers=headers)
+ urllib.request.urlopen(req)
+
+ def getPull(self, project, number):
+ pr = self.pull_requests[number - 1]
+ data = {
+ 'number': number,
+ 'updated_at': pr.updated_at,
+ 'base': {
+ 'repo': {
+ 'full_name': pr.project
+ },
+ 'ref': pr.branch,
+ },
+ 'mergeable': True,
+ 'head': {
+ 'sha': pr.head_sha
+ }
+ }
+ return data
+
+ def getGitUrl(self, project):
+ return os.path.join(self.upstream_root, str(project))
+
+ def real_getGitUrl(self, project):
+ return super(FakeGithubConnection, self).getGitUrl(project)
+
+ def getProjectBranches(self, project):
+ """Masks getProjectBranches since we don't have a real github"""
+
+ # just returns master for now
+ return ['master']
+
+ def commentPull(self, project, pr_number, message):
+ pull_request = self.pull_requests[pr_number - 1]
+ pull_request.addComment(message)
+
+ def mergePull(self, project, pr_number, sha=None):
+ pull_request = self.pull_requests[pr_number - 1]
+ if self.merge_failure:
+ raise Exception('Pull request was not merged')
+ if self.merge_not_allowed_count > 0:
+ self.merge_not_allowed_count -= 1
+ raise MergeFailure('Merge was not successful due to mergeability'
+ ' conflict')
+ pull_request.is_merged = True
+
+ def setCommitStatus(self, project, sha, state,
+ url='', description='', context=''):
+ owner, proj = project.split('/')
+ for pr in self.pull_requests:
+ pr_owner, pr_project = pr.project.split('/')
+ if (pr_owner == owner and pr_project == proj and
+ pr.head_sha == sha):
+ pr.setStatus(state, url, description, context)
+
+ def labelPull(self, project, pr_number, label):
+ pull_request = self.pull_requests[pr_number - 1]
+ pull_request.addLabel(label)
+
+ def unlabelPull(self, project, pr_number, label):
+ pull_request = self.pull_requests[pr_number - 1]
+ pull_request.removeLabel(label)
+
+
class BuildHistory(object):
def __init__(self, **kw):
self.__dict__.update(kw)
@@ -701,7 +1023,7 @@
"""
for change in changes:
- hostname = change.gerrit.canonical_hostname
+ hostname = change.source.canonical_hostname
path = os.path.join(self.jobdir.src_root, hostname, change.project)
try:
repo = git.Repo(path)
@@ -1451,6 +1773,16 @@
'zuul.driver.gerrit.GerritDriver.getConnection',
getGerritConnection))
+ def getGithubConnection(driver, name, config):
+ con = FakeGithubConnection(driver, name, config,
+ upstream_root=self.upstream_root)
+ setattr(self, 'fake_' + name, con)
+ return con
+
+ self.useFixture(fixtures.MonkeyPatch(
+ 'zuul.driver.github.GithubDriver.getConnection',
+ getGithubConnection))
+
# Set up smtp related fakes
# TODO(jhesketh): This should come from lib.connections for better
# coverage
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
index 92c66d1..1f8fdf3 100644
--- a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
@@ -13,3 +13,10 @@
- zuul.executor.hostname is defined
- zuul.executor.src_root is defined
- zuul.executor.log_root is defined
+
+ - name: Assert zuul.project variables are valid.
+ assert:
+ that:
+ - zuul.project.name == 'org/project'
+ - zuul.project.canonical_hostname == 'review.example.com'
+ - zuul.project.canonical_name == 'review.example.com/org/project'
diff --git a/tests/fixtures/config/in-repo/git/org_project1/README b/tests/fixtures/config/in-repo/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/in-repo/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/in-repo/main.yaml b/tests/fixtures/config/in-repo/main.yaml
index 208e274..5f57245 100644
--- a/tests/fixtures/config/in-repo/main.yaml
+++ b/tests/fixtures/config/in-repo/main.yaml
@@ -6,3 +6,4 @@
- common-config
untrusted-projects:
- org/project
+ - org/project1
diff --git a/tests/fixtures/layouts/basic-github.yaml b/tests/fixtures/layouts/basic-github.yaml
new file mode 100644
index 0000000..709fd02
--- /dev/null
+++ b/tests/fixtures/layouts/basic-github.yaml
@@ -0,0 +1,30 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action:
+ - opened
+ - changed
+ - reopened
+ branch: '^master$'
+ - event: pull_request
+ action: comment
+ comment: 'test me'
+ success:
+ github: {}
+ failure:
+ github: {}
+
+- job:
+ name: project-test1
+- job:
+ name: project-test2
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ - project-test2
diff --git a/tests/fixtures/layouts/dependent-github.yaml b/tests/fixtures/layouts/dependent-github.yaml
new file mode 100644
index 0000000..46cc7b3
--- /dev/null
+++ b/tests/fixtures/layouts/dependent-github.yaml
@@ -0,0 +1,35 @@
+- pipeline:
+ name: gate
+ description: Gatekeeping
+ manager: dependent
+ trigger:
+ github:
+ - event: pull_request
+ action: labeled
+ label: 'merge'
+ success:
+ github:
+ merge: true
+ unlabel: 'merge'
+ failure:
+ github:
+ unlabel: 'merge'
+
+- job:
+ name: project-test1
+- job:
+ name: project-test2
+- job:
+ name: project-merge
+ failure-message: Unable to merge change
+ hold-following-changes: true
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
diff --git a/tests/fixtures/layouts/dequeue-github.yaml b/tests/fixtures/layouts/dequeue-github.yaml
new file mode 100644
index 0000000..25e92c9
--- /dev/null
+++ b/tests/fixtures/layouts/dequeue-github.yaml
@@ -0,0 +1,18 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action:
+ - opened
+ - changed
+
+- job:
+ name: one-job-project-merge
+
+- project:
+ name: org/one-job-project
+ check:
+ jobs:
+ - one-job-project-merge
diff --git a/tests/fixtures/layouts/labeling-github.yaml b/tests/fixtures/layouts/labeling-github.yaml
new file mode 100644
index 0000000..33ce993
--- /dev/null
+++ b/tests/fixtures/layouts/labeling-github.yaml
@@ -0,0 +1,29 @@
+- pipeline:
+ name: labels
+ description: Trigger on labels
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: labeled
+ label:
+ - 'test'
+ - event: pull_request
+ action: unlabeled
+ label:
+ - 'do not test'
+ success:
+ github:
+ label:
+ - 'tests passed'
+ unlabel:
+ - 'test'
+
+- job:
+ name: project-labels
+
+- project:
+ name: org/project
+ labels:
+ jobs:
+ - project-labels
diff --git a/tests/fixtures/layouts/merging-github.yaml b/tests/fixtures/layouts/merging-github.yaml
new file mode 100644
index 0000000..4e13063
--- /dev/null
+++ b/tests/fixtures/layouts/merging-github.yaml
@@ -0,0 +1,19 @@
+- pipeline:
+ name: merge
+ description: Pipeline for merging the pull request
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: comment
+ comment: 'merge me'
+ success:
+ github:
+ merge: true
+ comment: false
+
+- project:
+ name: org/project
+ merge:
+ jobs:
+ - noop
diff --git a/tests/fixtures/layouts/push-tag-github.yaml b/tests/fixtures/layouts/push-tag-github.yaml
new file mode 100644
index 0000000..54683e9
--- /dev/null
+++ b/tests/fixtures/layouts/push-tag-github.yaml
@@ -0,0 +1,29 @@
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ github:
+ - event: push
+ ref: '^refs/heads/master$'
+
+- pipeline:
+ name: tag
+ manager: independent
+ trigger:
+ github:
+ - event: push
+ ref: ^refs/tags/.*$
+
+- job:
+ name: project-post
+- job:
+ name: project-tag
+
+- project:
+ name: org/project
+ post:
+ jobs:
+ - project-post
+ tag:
+ jobs:
+ - project-tag
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
new file mode 100644
index 0000000..bcbac1b
--- /dev/null
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -0,0 +1,45 @@
+- pipeline:
+ name: check
+ description: Standard check
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: opened
+ start:
+ github:
+ status: 'pending'
+ comment: false
+ success:
+ github:
+ status: 'success'
+
+- pipeline:
+ name: reporting
+ description: Uncommon reporting
+ manager: independent
+ trigger:
+ github:
+ - event: pull_request
+ action: comment
+ comment: 'reporting check'
+ start:
+ github: {}
+ success:
+ github:
+ comment: false
+ failure:
+ github:
+ comment: false
+
+- job:
+ name: project-test1
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ reporting:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf
new file mode 100644
index 0000000..58c7cd5
--- /dev/null
+++ b/tests/fixtures/zuul-github-driver.conf
@@ -0,0 +1,22 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+job_name_in_report=true
+status_url=http://zuul.example.com/status
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection github]
+driver=github
+
+[connection github_ssh]
+driver=github
+sshkey=/home/zuul/.ssh/id_rsa
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
new file mode 100644
index 0000000..3f567d2
--- /dev/null
+++ b/tests/unit/test_github_driver.py
@@ -0,0 +1,461 @@
+# Copyright 2015 GoodData
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import re
+from testtools.matchers import MatchesRegex
+import time
+
+from tests.base import ZuulTestCase, simple_layout, random_sha1
+
+logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(name)-32s '
+ '%(levelname)-8s %(message)s')
+
+
+class TestGithubDriver(ZuulTestCase):
+ config_file = 'zuul-github-driver.conf'
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_pull_event(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+
+ build_params = self.builds[0].parameters
+ self.assertEqual('master', build_params['ZUUL_BRANCH'])
+ self.assertEqual(str(A.number), build_params['ZUUL_CHANGE'])
+ self.assertEqual(A.head_sha, build_params['ZUUL_PATCHSET'])
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-test1').result)
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-test2').result)
+
+ job = self.getJobFromHistory('project-test2')
+ zuulvars = job.parameters['vars']['zuul']
+ self.assertEqual(A.number, zuulvars['change'])
+ self.assertEqual(A.head_sha, zuulvars['patchset'])
+ self.assertEqual(1, len(A.comments))
+ self.assertEqual(2, len(self.history))
+
+ # test_pull_unmatched_branch_event(self):
+ self.create_branch('org/project', 'unmatched_branch')
+ B = self.fake_github.openFakePullRequest(
+ 'org/project', 'unmatched_branch', 'B')
+ self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+
+ self.assertEqual(2, len(self.history))
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_comment_event(self):
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getCommentAddedEvent('test me'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+
+ # Test an unmatched comment, history should remain the same
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ self.fake_github.emitEvent(B.getCommentAddedEvent('casual comment'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+
+ @simple_layout('layouts/push-tag-github.yaml', driver='github')
+ def test_tag_event(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ sha = random_sha1()
+ self.fake_github.emitEvent(
+ self.fake_github.getPushEvent('org/project', 'refs/tags/newtag',
+ new_rev=sha))
+ self.waitUntilSettled()
+
+ build_params = self.builds[0].parameters
+ self.assertEqual('refs/tags/newtag', build_params['ZUUL_REF'])
+ self.assertEqual('00000000000000000000000000000000',
+ build_params['ZUUL_OLDREV'])
+ self.assertEqual(sha, build_params['ZUUL_NEWREV'])
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-tag').result)
+
+ @simple_layout('layouts/push-tag-github.yaml', driver='github')
+ def test_push_event(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ old_sha = random_sha1()
+ new_sha = random_sha1()
+ self.fake_github.emitEvent(
+ self.fake_github.getPushEvent('org/project', 'refs/heads/master',
+ old_sha, new_sha))
+ self.waitUntilSettled()
+
+ build_params = self.builds[0].parameters
+ self.assertEqual('refs/heads/master', build_params['ZUUL_REF'])
+ self.assertEqual(old_sha, build_params['ZUUL_OLDREV'])
+ self.assertEqual(new_sha, build_params['ZUUL_NEWREV'])
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-post').result)
+ self.assertEqual(1, len(self.history))
+
+ # test unmatched push event
+ old_sha = random_sha1()
+ new_sha = random_sha1()
+ self.fake_github.emitEvent(
+ self.fake_github.getPushEvent('org/project',
+ 'refs/heads/unmatched_branch',
+ old_sha, new_sha))
+ self.waitUntilSettled()
+
+ self.assertEqual(1, len(self.history))
+
+ @simple_layout('layouts/labeling-github.yaml', driver='github')
+ def test_labels(self):
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.addLabel('test'))
+ self.waitUntilSettled()
+ self.assertEqual(1, len(self.history))
+ self.assertEqual('project-labels', self.history[0].name)
+ self.assertEqual(['tests passed'], A.labels)
+
+ # test label removed
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ B.addLabel('do not test')
+ self.fake_github.emitEvent(B.removeLabel('do not test'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+ self.assertEqual('project-labels', self.history[1].name)
+ self.assertEqual(['tests passed'], B.labels)
+
+ # test unmatched label
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+ self.fake_github.emitEvent(C.addLabel('other label'))
+ self.waitUntilSettled()
+ self.assertEqual(2, len(self.history))
+ self.assertEqual(['other label'], C.labels)
+
+ @simple_layout('layouts/dequeue-github.yaml', driver='github')
+ def test_dequeue_pull_synchronized(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest(
+ 'org/one-job-project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+
+ # event update stamp has resolution one second, wait so the latter
+ # one has newer timestamp
+ time.sleep(1)
+ A.addCommit()
+ self.fake_github.emitEvent(A.getPullRequestSynchronizeEvent())
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(2, len(self.history))
+ self.assertEqual(1, self.countJobResults(self.history, 'ABORTED'))
+
+ @simple_layout('layouts/dequeue-github.yaml', driver='github')
+ def test_dequeue_pull_abandoned(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest(
+ 'org/one-job-project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+ self.fake_github.emitEvent(A.getPullRequestClosedEvent())
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(1, len(self.history))
+ self.assertEqual(1, self.countJobResults(self.history, 'ABORTED'))
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_git_https_url(self):
+ """Test that git_ssh option gives git url with ssh"""
+ url = self.fake_github.real_getGitUrl('org/project')
+ self.assertEqual('https://github.com/org/project', url)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_git_ssh_url(self):
+ """Test that git_ssh option gives git url with ssh"""
+ url = self.fake_github_ssh.real_getGitUrl('org/project')
+ self.assertEqual('ssh://git@github.com/org/project.git', url)
+
+ @simple_layout('layouts/reporting-github.yaml', driver='github')
+ def test_reporting(self):
+ # pipeline reports pull status both on start and success
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+ self.waitUntilSettled()
+ self.assertIn('check', A.statuses)
+ check_status = A.statuses['check']
+ self.assertEqual('Standard check', check_status['description'])
+ self.assertEqual('pending', check_status['state'])
+ self.assertEqual('http://zuul.example.com/status', check_status['url'])
+ self.assertEqual(0, len(A.comments))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ check_status = A.statuses['check']
+ self.assertEqual('Standard check', check_status['description'])
+ self.assertEqual('success', check_status['state'])
+ self.assertEqual('http://zuul.example.com/status', check_status['url'])
+ self.assertEqual(1, len(A.comments))
+ self.assertThat(A.comments[0],
+ MatchesRegex('.*Build succeeded.*', re.DOTALL))
+
+ # pipeline does not report any status but does comment
+ self.executor_server.hold_jobs_in_build = True
+ self.fake_github.emitEvent(
+ A.getCommentAddedEvent('reporting check'))
+ self.waitUntilSettled()
+ self.assertNotIn('reporting', A.statuses)
+ # comments increased by one for the start message
+ self.assertEqual(2, len(A.comments))
+ self.assertThat(A.comments[1],
+ MatchesRegex('.*Starting reporting jobs.*', re.DOTALL))
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertNotIn('reporting', A.statuses)
+ self.assertEqual(2, len(A.comments))
+
+ @simple_layout('layouts/merging-github.yaml', driver='github')
+ def test_report_pull_merge(self):
+ # pipeline merges the pull request on success
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertTrue(A.is_merged)
+
+ # pipeline merges the pull request on success after failure
+ self.fake_github.merge_failure = True
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ self.fake_github.emitEvent(B.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertFalse(B.is_merged)
+ self.fake_github.merge_failure = False
+
+ # pipeline merges the pull request on second run of merge
+ # first merge failed on 405 Method Not Allowed error
+ self.fake_github.merge_not_allowed_count = 1
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+ self.fake_github.emitEvent(C.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertTrue(C.is_merged)
+
+ # pipeline does not merge the pull request
+ # merge failed on 405 Method Not Allowed error - twice
+ self.fake_github.merge_not_allowed_count = 2
+ D = self.fake_github.openFakePullRequest('org/project', 'master', 'D')
+ self.fake_github.emitEvent(D.getCommentAddedEvent('merge me'))
+ self.waitUntilSettled()
+ self.assertFalse(D.is_merged)
+
+ @simple_layout('layouts/dependent-github.yaml', driver='github')
+ def test_parallel_changes(self):
+ "Test that changes are tested in parallel and merged in series"
+
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+
+ self.fake_github.emitEvent(A.addLabel('merge'))
+ self.fake_github.emitEvent(B.addLabel('merge'))
+ self.fake_github.emitEvent(C.addLabel('merge'))
+
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 1)
+ self.assertEqual(self.builds[0].name, 'project-merge')
+ self.assertTrue(self.builds[0].hasChanges(A))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 3)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertTrue(self.builds[1].hasChanges(A))
+ self.assertEqual(self.builds[2].name, 'project-merge')
+ self.assertTrue(self.builds[2].hasChanges(A, B))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 5)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertTrue(self.builds[1].hasChanges(A))
+
+ self.assertEqual(self.builds[2].name, 'project-test1')
+ self.assertTrue(self.builds[2].hasChanges(A))
+ self.assertEqual(self.builds[3].name, 'project-test2')
+ self.assertTrue(self.builds[3].hasChanges(A, B))
+
+ self.assertEqual(self.builds[4].name, 'project-merge')
+ self.assertTrue(self.builds[4].hasChanges(A, B, C))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 6)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertTrue(self.builds[0].hasChanges(A))
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertTrue(self.builds[1].hasChanges(A))
+
+ self.assertEqual(self.builds[2].name, 'project-test1')
+ self.assertTrue(self.builds[2].hasChanges(A, B))
+ self.assertEqual(self.builds[3].name, 'project-test2')
+ self.assertTrue(self.builds[3].hasChanges(A, B))
+
+ self.assertEqual(self.builds[4].name, 'project-test1')
+ self.assertTrue(self.builds[4].hasChanges(A, B, C))
+ self.assertEqual(self.builds[5].name, 'project-test2')
+ self.assertTrue(self.builds[5].hasChanges(A, B, C))
+
+ all_builds = self.builds[:]
+ self.release(all_builds[2])
+ self.release(all_builds[3])
+ self.waitUntilSettled()
+ self.assertFalse(A.is_merged)
+ self.assertFalse(B.is_merged)
+ self.assertFalse(C.is_merged)
+
+ self.release(all_builds[0])
+ self.release(all_builds[1])
+ self.waitUntilSettled()
+ self.assertTrue(A.is_merged)
+ self.assertTrue(B.is_merged)
+ self.assertFalse(C.is_merged)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), 9)
+ self.assertTrue(C.is_merged)
+
+ self.assertNotIn('merge', A.labels)
+ self.assertNotIn('merge', B.labels)
+ self.assertNotIn('merge', C.labels)
+
+ @simple_layout('layouts/dependent-github.yaml', driver='github')
+ def test_failed_changes(self):
+ "Test that a change behind a failed change is retested"
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+
+ self.executor_server.failJob('project-test1', A)
+
+ self.fake_github.emitEvent(A.addLabel('merge'))
+ self.fake_github.emitEvent(B.addLabel('merge'))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+
+ self.waitUntilSettled()
+ # It's certain that the merge job for change 2 will run, but
+ # the test1 and test2 jobs may or may not run.
+ self.assertTrue(len(self.history) > 6)
+ self.assertFalse(A.is_merged)
+ self.assertTrue(B.is_merged)
+ self.assertNotIn('merge', A.labels)
+ self.assertNotIn('merge', B.labels)
+
+ @simple_layout('layouts/dependent-github.yaml', driver='github')
+ def test_failed_change_at_head(self):
+ "Test that if a change at the head fails, jobs behind it are canceled"
+
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('org/project', 'master', 'B')
+ C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
+
+ self.executor_server.failJob('project-test1', A)
+
+ self.fake_github.emitEvent(A.addLabel('merge'))
+ self.fake_github.emitEvent(B.addLabel('merge'))
+ self.fake_github.emitEvent(C.addLabel('merge'))
+
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 1)
+ self.assertEqual(self.builds[0].name, 'project-merge')
+ self.assertTrue(self.builds[0].hasChanges(A))
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 6)
+ self.assertEqual(self.builds[0].name, 'project-test1')
+ self.assertEqual(self.builds[1].name, 'project-test2')
+ self.assertEqual(self.builds[2].name, 'project-test1')
+ self.assertEqual(self.builds[3].name, 'project-test2')
+ self.assertEqual(self.builds[4].name, 'project-test1')
+ self.assertEqual(self.builds[5].name, 'project-test2')
+
+ self.release(self.builds[0])
+ self.waitUntilSettled()
+
+ # project-test2, project-merge for B
+ self.assertEqual(len(self.builds), 2)
+ self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), 15)
+ self.assertFalse(A.is_merged)
+ self.assertTrue(B.is_merged)
+ self.assertTrue(C.is_merged)
+ self.assertNotIn('merge', A.labels)
+ self.assertNotIn('merge', B.labels)
+ self.assertNotIn('merge', C.labels)
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index c4ed656..5959889 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2164,12 +2164,6 @@
self.assertEqual(set(['project-test-nomatch-starts-empty',
'project-test-nomatch-starts-full']), run_jobs)
- @skip("Disabled for early v3 development")
- def test_test_config(self):
- "Test that we can test the config"
- self.sched.testConfig(self.config.get('zuul', 'tenant_config'),
- self.connections)
-
def test_queue_names(self):
"Test shared change queue names"
tenant = self.sched.abide.tenants.get('tenant-one')
diff --git a/tests/unit/test_scheduler_cmd.py b/tests/unit/test_scheduler_cmd.py
new file mode 100644
index 0000000..ee6200f
--- /dev/null
+++ b/tests/unit/test_scheduler_cmd.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import testtools
+import zuul.cmd.scheduler
+
+from tests import base
+
+
+class TestSchedulerCmdArguments(testtools.TestCase):
+
+ def setUp(self):
+ super(TestSchedulerCmdArguments, self).setUp()
+ self.app = zuul.cmd.scheduler.Scheduler()
+
+ def test_test_config(self):
+ conf_path = os.path.join(base.FIXTURE_DIR, 'zuul.conf')
+ self.app.parse_arguments(['-t', '-c', conf_path])
+ self.assertTrue(self.app.args.validate)
+ self.app.read_config()
+ self.assertEqual(0, self.app.test_config())
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 3919418..2168a7f 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -191,6 +191,61 @@
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='3,1')])
+ def test_crd_dynamic_config_branch(self):
+ # Test that we can create a job in one repo and be able to use
+ # it from a different branch on a different repo.
+
+ self.create_branch('org/project1', 'stable')
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test2
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - project-test2
+ """)
+
+ in_repo_playbook = textwrap.dedent(
+ """
+ - hosts: all
+ tasks: []
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf,
+ 'playbooks/project-test2.yaml': in_repo_playbook}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+
+ second_repo_conf = textwrap.dedent(
+ """
+ - project:
+ name: org/project1
+ check:
+ jobs:
+ - project-test2
+ """)
+
+ second_file_dict = {'.zuul.yaml': second_repo_conf}
+ B = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'B',
+ files=second_file_dict)
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 1, "A should report")
+ self.assertHistory([
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
+ ])
+
def test_untrusted_syntax_error(self):
in_repo_conf = textwrap.dedent(
"""
@@ -253,6 +308,26 @@
self.assertIn('syntax error', A.messages[1],
"A should have a syntax error reported")
+ def test_untrusted_shadow_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: common-config-test
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 2,
+ "A should report start and failure")
+ self.assertIn('not permitted to shadow', A.messages[1],
+ "A should have a syntax error reported")
+
class TestAnsible(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index f1d1015..5328bba 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -40,7 +40,7 @@
super(Scheduler, self).__init__()
self.gear_server_pid = None
- def parse_arguments(self):
+ def parse_arguments(self, args=None):
parser = argparse.ArgumentParser(description='Project gating system.')
parser.add_argument('-c', dest='config',
help='specify the config file')
@@ -52,7 +52,7 @@
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
- self.args = parser.parse_args()
+ self.args = parser.parse_args(args)
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d7cef94..070e731 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -65,6 +65,8 @@
def configuration_exceptions(stanza, conf):
try:
yield
+ except ConfigurationSyntaxError:
+ raise
except Exception as e:
conf = copy.deepcopy(conf)
context = conf.pop('_source_context')
@@ -271,7 +273,29 @@
]
@staticmethod
- def fromYaml(tenant, layout, conf):
+ def _getImpliedBranches(reference, job, project_pipeline):
+ # If the current job definition is not in the same branch as
+ # the reference definition of this job, and this is a project
+ # repo, add an implicit branch matcher for this branch
+ # (assuming there are no explicit branch matchers). But only
+ # for top-level job definitions and variants.
+ # Project-pipeline job variants should more closely attach to
+ # their branch if they appear in a project-repo.
+ if (reference and
+ reference.source_context and
+ reference.source_context.branch != job.source_context.branch):
+ same_context = False
+ else:
+ same_context = True
+
+ if (job.source_context and
+ (not job.source_context.trusted) and
+ ((not same_context) or project_pipeline)):
+ return [job.source_context.branch]
+ return None
+
+ @staticmethod
+ def fromYaml(tenant, layout, conf, project_pipeline=False):
with configuration_exceptions('job', conf):
JobParser.getSchema()(conf)
@@ -280,6 +304,8 @@
# them (e.g., "job.run = ..." rather than
# "job.run.append(...)").
+ reference = layout.jobs.get(conf['name'], [None])[0]
+
job = model.Job(conf['name'])
job.source_context = conf.get('_source_context')
if 'auth' in conf:
@@ -316,9 +342,10 @@
run = model.PlaybookContext(job.source_context, run_name)
job.run = (run,)
else:
- run_name = os.path.join('playbooks', job.name)
- run = model.PlaybookContext(job.source_context, run_name)
- job.implied_run = (run,) + job.implied_run
+ if not project_pipeline:
+ run_name = os.path.join('playbooks', job.name)
+ run = model.PlaybookContext(job.source_context, run_name)
+ job.implied_run = (run,) + job.implied_run
for k in JobParser.simple_attributes:
a = k.replace('-', '_')
@@ -350,13 +377,14 @@
job.dependencies = frozenset(as_list(conf.get('dependencies')))
- roles = []
- for role in conf.get('roles', []):
- if 'zuul' in role:
- r = JobParser._makeZuulRole(tenant, job, role)
- if r:
- roles.append(r)
- job.roles = job.roles.union(set(roles))
+ if 'roles' in conf:
+ roles = []
+ for role in conf.get('roles', []):
+ if 'zuul' in role:
+ r = JobParser._makeZuulRole(tenant, job, role)
+ if r:
+ roles.append(r)
+ job.roles = job.roles.union(set(roles))
variables = conf.get('vars', None)
if variables:
@@ -372,14 +400,20 @@
allowed.append(project.name)
job.allowed_projects = frozenset(allowed)
- # If the definition for this job came from a project repo,
- # implicitly apply a branch matcher for the branch it was on.
- if (not job.source_context.trusted):
- branches = [job.source_context.branch]
- elif 'branches' in conf:
+ # If the current job definition is not in the same branch as
+ # the reference definition of this job, and this is a project
+ # repo, add an implicit branch matcher for this branch
+ # (assuming there are no explicit branch matchers). But only
+ # for top-level job definitions and variants.
+ # Project-pipeline job variants should more closely attach to
+ # their branch if they appear in a project-repo.
+
+ branches = None
+ if (project_pipeline or 'branches' not in conf):
+ branches = JobParser._getImpliedBranches(
+ reference, job, project_pipeline)
+ if (not branches) and ('branches' in conf):
branches = as_list(conf['branches'])
- else:
- branches = None
if branches:
matchers = []
for branch in branches:
@@ -408,7 +442,7 @@
return model.ZuulRole(role.get('name', name),
project.connection_name,
- project.name, trusted)
+ project.name)
class ProjectTemplateParser(object):
@@ -456,23 +490,22 @@
start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, six.string_types):
- job = model.Job(conf_job)
- job_list.addJob(job)
+ attrs = dict(name=conf_job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params
jobname, attrs = conf_job.items()[0]
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
- attrs['_source_context'] = source_context
- attrs['_start_mark'] = start_mark
- job_list.addJob(JobParser.fromYaml(tenant, layout, attrs))
else:
# Not overriding, so add a blank job
- job = model.Job(jobname)
- job_list.addJob(job)
+ attrs = dict(name=jobname)
else:
raise Exception("Job must be a string or dictionary")
+ attrs['_source_context'] = source_context
+ attrs['_start_mark'] = start_mark
+ job_list.addJob(JobParser.fromYaml(tenant, layout, attrs,
+ project_pipeline=True))
class ProjectParser(object):
@@ -993,7 +1026,8 @@
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in data.jobs:
- layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
+ with configuration_exceptions('job', config_job):
+ layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
for config_semaphore in data.semaphores:
layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
@@ -1122,7 +1156,8 @@
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in config.jobs:
- layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
+ with configuration_exceptions('job', config_job):
+ layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
for config_template in config.project_templates:
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
diff --git a/zuul/driver/github/__init__.py b/zuul/driver/github/__init__.py
new file mode 100644
index 0000000..2d6829d
--- /dev/null
+++ b/zuul/driver/github/__init__.py
@@ -0,0 +1,43 @@
+# Copyright 2017 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from zuul.driver import Driver, ConnectionInterface, TriggerInterface
+from zuul.driver import SourceInterface
+import githubconnection
+import githubtrigger
+import githubsource
+import githubreporter
+
+
+class GithubDriver(Driver, ConnectionInterface, TriggerInterface,
+ SourceInterface):
+ name = 'github'
+
+ def getConnection(self, name, config):
+ return githubconnection.GithubConnection(self, name, config)
+
+ def getTrigger(self, connection, config=None):
+ return githubtrigger.GithubTrigger(self, connection, config)
+
+ def getSource(self, connection):
+ return githubsource.GithubSource(self, connection)
+
+ def getReporter(self, connection, config=None):
+ return githubreporter.GithubReporter(self, connection, config)
+
+ def getTriggerSchema(self):
+ return githubtrigger.getSchema()
+
+ def getReporterSchema(self):
+ return githubreporter.getSchema()
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
new file mode 100644
index 0000000..f2a8fc0
--- /dev/null
+++ b/zuul/driver/github/githubconnection.py
@@ -0,0 +1,348 @@
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import hmac
+import hashlib
+import time
+
+import webob
+import webob.dec
+import voluptuous as v
+import github3
+from github3.exceptions import MethodNotAllowed
+
+from zuul.connection import BaseConnection
+from zuul.model import PullRequest, Ref, GithubTriggerEvent
+from zuul.exceptions import MergeFailure
+
+
+class GithubWebhookListener():
+
+ log = logging.getLogger("zuul.GithubWebhookListener")
+
+ def __init__(self, connection):
+ self.connection = connection
+
+ def handle_request(self, path, tenant_name, request):
+ if request.method != 'POST':
+ self.log.debug("Only POST method is allowed.")
+ raise webob.exc.HTTPMethodNotAllowed(
+ 'Only POST method is allowed.')
+
+ self.log.debug("Github Webhook Received.")
+
+ self._validate_signature(request)
+
+ self.__dispatch_event(request)
+
+ def __dispatch_event(self, request):
+ try:
+ event = request.headers['X-Github-Event']
+ self.log.debug("X-Github-Event: " + event)
+ except KeyError:
+ self.log.debug("Request headers missing the X-Github-Event.")
+ raise webob.exc.HTTPBadRequest('Please specify a X-Github-Event '
+ 'header.')
+
+ try:
+ method = getattr(self, '_event_' + event)
+ except AttributeError:
+ message = "Unhandled X-Github-Event: {0}".format(event)
+ self.log.debug(message)
+ raise webob.exc.HTTPBadRequest(message)
+
+ try:
+ event = method(request)
+ except:
+ self.log.exception('Exception when handling event:')
+
+ if event:
+ event.project_hostname = self.connection.canonical_hostname
+ self.log.debug('Scheduling github event: {0}'.format(event.type))
+ self.connection.sched.addEvent(event)
+
+ def _event_push(self, request):
+ body = request.json_body
+ base_repo = body.get('repository')
+
+ event = GithubTriggerEvent()
+ event.trigger_name = 'github'
+ event.project_name = base_repo.get('full_name')
+ event.type = 'push'
+
+ event.ref = body.get('ref')
+ event.oldrev = body.get('before')
+ event.newrev = body.get('after')
+
+ ref_parts = event.ref.split('/') # ie, ['refs', 'heads', 'master']
+
+ if ref_parts[1] == "heads":
+ # necessary for the scheduler to match against particular branches
+ event.branch = ref_parts[2]
+
+ return event
+
+ def _event_pull_request(self, request):
+ body = request.json_body
+ action = body.get('action')
+ pr_body = body.get('pull_request')
+
+ event = self._pull_request_to_event(pr_body)
+
+ event.type = 'pull_request'
+ if action == 'opened':
+ event.action = 'opened'
+ elif action == 'synchronize':
+ event.action = 'changed'
+ elif action == 'closed':
+ event.action = 'closed'
+ elif action == 'reopened':
+ event.action = 'reopened'
+ elif action == 'labeled':
+ event.action = 'labeled'
+ event.label = body['label']['name']
+ elif action == 'unlabeled':
+ event.action = 'unlabeled'
+ event.label = body['label']['name']
+ else:
+ return None
+
+ return event
+
+ def _event_issue_comment(self, request):
+ """Handles pull request comments"""
+ body = request.json_body
+ action = body.get('action')
+ if action != 'created':
+ return
+ pr_body = self._issue_to_pull_request(body)
+ number = body.get('issue').get('number')
+ project_name = body.get('repository').get('full_name')
+ pr_body = self.connection.getPull(project_name, number)
+ if pr_body is None:
+ return
+
+ event = self._pull_request_to_event(pr_body)
+ event.comment = body.get('comment').get('body')
+ event.type = 'pull_request'
+ event.action = 'comment'
+ return event
+
+ def _issue_to_pull_request(self, body):
+ number = body.get('issue').get('number')
+ project_name = body.get('repository').get('full_name')
+ pr_body = self.connection.getPull(project_name, number)
+ if pr_body is None:
+ self.log.debug('Pull request #%s not found in project %s' %
+ (number, project_name))
+ return pr_body
+
+ def _validate_signature(self, request):
+ secret = self.connection.connection_config.get('webhook_token', None)
+ if secret is None:
+ return True
+
+ body = request.body
+ try:
+ request_signature = request.headers['X-Hub-Signature']
+ except KeyError:
+ raise webob.exc.HTTPUnauthorized(
+ 'Please specify a X-Hub-Signature header with secret.')
+
+ payload_signature = 'sha1=' + hmac.new(secret,
+ body,
+ hashlib.sha1).hexdigest()
+
+ self.log.debug("Payload Signature: {0}".format(str(payload_signature)))
+ self.log.debug("Request Signature: {0}".format(str(request_signature)))
+ if str(payload_signature) != str(request_signature):
+ raise webob.exc.HTTPUnauthorized(
+ 'Request signature does not match calculated payload '
+ 'signature. Check that secret is correct.')
+
+ return True
+
+ def _pull_request_to_event(self, pr_body):
+ event = GithubTriggerEvent()
+ event.trigger_name = 'github'
+
+ base = pr_body.get('base')
+ base_repo = base.get('repo')
+ head = pr_body.get('head')
+
+ event.project_name = base_repo.get('full_name')
+ event.change_number = pr_body.get('number')
+ event.change_url = self.connection.getPullUrl(event.project_name,
+ event.change_number)
+ event.updated_at = pr_body.get('updated_at')
+ event.branch = base.get('ref')
+ event.refspec = "refs/pull/" + str(pr_body.get('number')) + "/head"
+ event.patch_number = head.get('sha')
+
+ return event
+
+
+class GithubConnection(BaseConnection):
+ driver_name = 'github'
+ log = logging.getLogger("connection.github")
+ payload_path = 'payload'
+ git_user = 'git'
+ git_host = 'github.com'
+
+ def __init__(self, driver, connection_name, connection_config):
+ super(GithubConnection, self).__init__(
+ driver, connection_name, connection_config)
+ self.github = None
+ self.canonical_hostname = self.connection_config.get(
+ 'canonical_hostname', 'github.com')
+ self._change_cache = {}
+ self.projects = {}
+ self.source = driver.getSource(self)
+ self._git_ssh = bool(self.connection_config.get('sshkey', None))
+
+ def onLoad(self):
+ webhook_listener = GithubWebhookListener(self)
+ self.registerHttpHandler(self.payload_path,
+ webhook_listener.handle_request)
+ self._authenticateGithubAPI()
+
+ def onStop(self):
+ self.unregisterHttpHandler(self.payload_path)
+
+ def _authenticateGithubAPI(self):
+ token = self.connection_config.get('api_token', None)
+ if token is not None:
+ self.github = github3.login(token=token)
+ self.log.info("Github API Authentication successful.")
+ else:
+ self.github = None
+ self.log.info(
+ "No Github credentials found in zuul configuration, cannot "
+ "authenticate.")
+
+ def maintainCache(self, relevant):
+ for key, change in self._change_cache.items():
+ if change not in relevant:
+ del self._change_cache[key]
+
+ def getChange(self, event):
+ """Get the change representing an event."""
+
+ project = self.source.getProject(event.project_name)
+ if event.change_number:
+ change = PullRequest(event.project_name)
+ change.project = project
+ change.number = event.change_number
+ change.refspec = event.refspec
+ change.branch = event.branch
+ change.url = event.change_url
+ change.updated_at = self._ghTimestampToDate(event.updated_at)
+ change.patchset = event.patch_number
+ elif event.ref:
+ change = Ref(project)
+ change.ref = event.ref
+ change.oldrev = event.oldrev
+ change.newrev = event.newrev
+ change.url = self.getGitwebUrl(project, sha=event.newrev)
+ else:
+ change = Ref(project)
+ return change
+
+ def getGitUrl(self, project):
+ if self._git_ssh:
+ url = 'ssh://%s@%s/%s.git' % \
+ (self.git_user, self.git_host, project)
+ else:
+ url = 'https://%s/%s' % (self.git_host, project)
+ return url
+
+ def getGitwebUrl(self, project, sha=None):
+ url = 'https://%s/%s' % (self.git_host, project)
+ if sha is not None:
+ url += '/commit/%s' % sha
+ return url
+
+ def getProject(self, name):
+ return self.projects.get(name)
+
+ def addProject(self, project):
+ self.projects[project.name] = project
+
+ def getProjectBranches(self, project):
+ owner, proj = project.name.split('/')
+ repository = self.github.repository(owner, proj)
+ branches = [branch.name for branch in repository.branches()]
+ return branches
+
+ def getPullUrl(self, project, number):
+ return '%s/pull/%s' % (self.getGitwebUrl(project), number)
+
+ def getPull(self, project_name, number):
+ owner, proj = project_name.split('/')
+ return self.github.pull_request(owner, proj, number).as_dict()
+
+ def canMerge(self, change, allow_needs):
+ # This API call may get a false (null) while GitHub is calculating
+ # if it can merge. The github3.py library will just return that as
+ # false. This could lead to false negatives.
+ # Additionally, this only checks if the PR code could merge
+ # cleanly to the target branch. It does not evaluate any branch
+ # protection merge requirements (such as reviews and status states)
+ # At some point in the future this may be available through the API
+ # or we can fetch the branch protection settings and evaluate within
+ # Zuul whether or not those protections have been met
+ # For now, just send back a True value.
+ return True
+
+ def commentPull(self, project, pr_number, message):
+ owner, proj = project.split('/')
+ repository = self.github.repository(owner, proj)
+ pull_request = repository.issue(pr_number)
+ pull_request.create_comment(message)
+
+ def mergePull(self, project, pr_number, sha=None):
+ owner, proj = project.split('/')
+ pull_request = self.github.pull_request(owner, proj, pr_number)
+ try:
+ result = pull_request.merge(sha=sha)
+ except MethodNotAllowed as e:
+ raise MergeFailure('Merge was not successful due to mergeability'
+ ' conflict, original error is %s' % e)
+ if not result:
+ raise Exception('Pull request was not merged')
+
+ def setCommitStatus(self, project, sha, state, url='', description='',
+ context=''):
+ owner, proj = project.split('/')
+ repository = self.github.repository(owner, proj)
+ repository.create_status(sha, state, url, description, context)
+
+ def labelPull(self, project, pr_number, label):
+ owner, proj = project.split('/')
+ pull_request = self.github.issue(owner, proj, pr_number)
+ pull_request.add_labels(label)
+
+ def unlabelPull(self, project, pr_number, label):
+ owner, proj = project.split('/')
+ pull_request = self.github.issue(owner, proj, pr_number)
+ pull_request.remove_label(label)
+
+ def _ghTimestampToDate(self, timestamp):
+ return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
+
+
+def getSchema():
+ github_connection = v.Any(str, v.Schema({}, extra=True))
+ return github_connection
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
new file mode 100644
index 0000000..159103c
--- /dev/null
+++ b/zuul/driver/github/githubreporter.py
@@ -0,0 +1,125 @@
+# Copyright 2015 Puppet Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+import time
+
+from zuul.reporter import BaseReporter
+from zuul.exceptions import MergeFailure
+
+
+class GithubReporter(BaseReporter):
+ """Sends off reports to Github."""
+
+ name = 'github'
+ log = logging.getLogger("zuul.GithubReporter")
+
+ def __init__(self, driver, connection, config=None):
+ super(GithubReporter, self).__init__(driver, connection, config)
+ self._commit_status = self.config.get('status', None)
+ self._create_comment = self.config.get('comment', True)
+ self._merge = self.config.get('merge', False)
+ self._labels = self.config.get('label', [])
+ if not isinstance(self._labels, list):
+ self._labels = [self._labels]
+ self._unlabels = self.config.get('unlabel', [])
+ if not isinstance(self._unlabels, list):
+ self._unlabels = [self._unlabels]
+
+ def report(self, source, pipeline, item):
+ """Comment on PR and set commit status."""
+ if self._create_comment:
+ self.addPullComment(pipeline, item)
+ if (self._commit_status is not None and
+ hasattr(item.change, 'patchset') and
+ item.change.patchset is not None):
+ self.setPullStatus(pipeline, item)
+ if (self._merge and
+ hasattr(item.change, 'number')):
+ self.mergePull(item)
+ if self._labels or self._unlabels:
+ self.setLabels(item)
+
+ def addPullComment(self, pipeline, item):
+ message = self._formatItemReport(pipeline, item)
+ project = item.change.project.name
+ pr_number = item.change.number
+ self.log.debug(
+ 'Reporting change %s, params %s, message: %s' %
+ (item.change, self.config, message))
+ self.connection.commentPull(project, pr_number, message)
+
+ def setPullStatus(self, pipeline, item):
+ project = item.change.project.name
+ sha = item.change.patchset
+ context = pipeline.name
+ state = self._commit_status
+ url = ''
+ if self.connection.sched.config.has_option('zuul', 'status_url'):
+ url = self.connection.sched.config.get('zuul', 'status_url')
+ description = ''
+ if pipeline.description:
+ description = pipeline.description
+
+ self.log.debug(
+ 'Reporting change %s, params %s, status:\n'
+ 'context: %s, state: %s, description: %s, url: %s' %
+ (item.change, self.config, context, state,
+ description, url))
+
+ self.connection.setCommitStatus(
+ project, sha, state, url, description, context)
+
+ def mergePull(self, item):
+ project = item.change.project.name
+ pr_number = item.change.number
+ sha = item.change.patchset
+ self.log.debug('Reporting change %s, params %s, merging via API' %
+ (item.change, self.config))
+ try:
+ self.connection.mergePull(project, pr_number, sha)
+ except MergeFailure:
+ time.sleep(2)
+ self.log.debug('Trying to merge change %s again...' % item.change)
+ self.connection.mergePull(project, pr_number, sha)
+ item.change.is_merged = True
+
+ def setLabels(self, item):
+ project = item.change.project.name
+ pr_number = item.change.number
+ if self._labels:
+ self.log.debug('Reporting change %s, params %s, labels:\n%s' %
+ (item.change, self.config, self._labels))
+ for label in self._labels:
+ self.connection.labelPull(project, pr_number, label)
+ if self._unlabels:
+ self.log.debug('Reporting change %s, params %s, unlabels:\n%s' %
+ (item.change, self.config, self._unlabels))
+ for label in self._unlabels:
+ self.connection.unlabelPull(project, pr_number, label)
+
+
+def getSchema():
+ def toList(x):
+ return v.Any([x], x)
+
+ github_reporter = v.Schema({
+ 'status': v.Any('pending', 'success', 'failure'),
+ 'comment': bool,
+ 'merge': bool,
+ 'label': toList(str),
+ 'unlabel': toList(str)
+ })
+ return github_reporter
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
new file mode 100644
index 0000000..a638122
--- /dev/null
+++ b/zuul/driver/github/githubsource.py
@@ -0,0 +1,88 @@
+# Copyright 2014 Puppet Labs Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import time
+
+from zuul.source import BaseSource
+from zuul.model import Project
+
+
+class GithubSource(BaseSource):
+ name = 'github'
+ log = logging.getLogger("zuul.source.GithubSource")
+
+ def __init__(self, driver, connection, config=None):
+ hostname = connection.canonical_hostname
+ super(GithubSource, self).__init__(driver, connection,
+ hostname, config)
+
+ def getRefSha(self, project, ref):
+ """Return a sha for a given project ref."""
+ raise NotImplementedError()
+
+ def waitForRefSha(self, project, ref, old_sha=''):
+ """Block until a ref shows up in a given project."""
+ raise NotImplementedError()
+
+ def isMerged(self, change, head=None):
+ """Determine if change is merged."""
+ if not change.number:
+ # Not a pull request, considering merged.
+ return True
+ return change.is_merged
+
+ def canMerge(self, change, allow_needs):
+ """Determine if change can merge."""
+
+ if not change.number:
+ # Not a pull request, considering merged.
+ return True
+ return self.connection.canMerge(change, allow_needs)
+
+ def postConfig(self):
+ """Called after configuration has been processed."""
+ pass
+
+ def getChange(self, event):
+ return self.connection.getChange(event)
+
+ def getProject(self, name):
+ p = self.connection.getProject(name)
+ if not p:
+ p = Project(name, self)
+ self.connection.addProject(p)
+ return p
+
+ def getProjectBranches(self, project):
+ return self.connection.getProjectBranches(project)
+
+ def getProjectOpenChanges(self, project):
+ """Get the open changes for a project."""
+ raise NotImplementedError()
+
+ def updateChange(self, change, history=None):
+ """Update information for a change."""
+ raise NotImplementedError()
+
+ def getGitUrl(self, project):
+ """Get the git url for a project."""
+ return self.connection.getGitUrl(project)
+
+ def getGitwebUrl(self, project, sha=None):
+ """Get the git-web url for a project."""
+ return self.connection.getGitwebUrl(project, sha)
+
+ def _ghTimestampToDate(self, timestamp):
+ return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
diff --git a/zuul/driver/github/githubtrigger.py b/zuul/driver/github/githubtrigger.py
new file mode 100644
index 0000000..541c783
--- /dev/null
+++ b/zuul/driver/github/githubtrigger.py
@@ -0,0 +1,69 @@
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+from zuul.model import EventFilter
+from zuul.trigger import BaseTrigger
+
+
+class GithubTrigger(BaseTrigger):
+ name = 'github'
+ log = logging.getLogger("zuul.trigger.GithubTrigger")
+
+ def getEventFilters(self, trigger_config):
+ def toList(item):
+ if not item:
+ return []
+ if isinstance(item, list):
+ return item
+ return [item]
+
+ efilters = []
+ for trigger in toList(trigger_config):
+ f = EventFilter(
+ trigger=self,
+ types=toList(trigger['event']),
+ actions=toList(trigger.get('action')),
+ branches=toList(trigger.get('branch')),
+ refs=toList(trigger.get('ref')),
+ comments=toList(trigger.get('comment')),
+ labels=toList(trigger.get('label')),
+ unlabels=toList(trigger.get('unlabel'))
+ )
+ efilters.append(f)
+
+ return efilters
+
+ def onPullRequest(self, payload):
+ pass
+
+
+def getSchema():
+ def toList(x):
+ return v.Any([x], x)
+
+ github_trigger = {
+ v.Required('event'):
+ toList(v.Any('pull_request',
+ 'push')),
+ 'action': toList(str),
+ 'branch': toList(str),
+ 'ref': toList(str),
+ 'comment': toList(str),
+ 'label': toList(str),
+ 'unlabel': toList(str),
+ }
+
+ return github_trigger
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 7e2d296..9f234e9 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -183,10 +183,15 @@
dependent_items.reverse()
# TODOv3(jeblair): This ansible vars data structure will
# replace the environment variables below.
+ project = dict(
+ name=item.change.project.name,
+ canonical_hostname=item.change.project.canonical_hostname,
+ canonical_name=item.change.project.canonical_name)
+
zuul_params = dict(uuid=uuid,
pipeline=pipeline.name,
job=job.name,
- project=item.change.project.name,
+ project=project,
tags=' '.join(sorted(job.tags)))
if hasattr(item.change, 'branch'):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index fa0f4d5..4801de2 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -108,7 +108,8 @@
self.pre_playbooks = []
self.post_playbooks = []
self.roles = []
- self.roles_path = []
+ self.trusted_roles_path = []
+ self.untrusted_roles_path = []
self.untrusted_config = os.path.join(
self.ansible_root, 'untrusted.cfg')
self.trusted_config = os.path.join(self.ansible_root, 'trusted.cfg')
@@ -142,6 +143,10 @@
count = len(self.roles)
root = os.path.join(self.ansible_root, 'role_%i' % (count,))
os.makedirs(root)
+ trusted = os.path.join(root, 'trusted')
+ os.makedirs(trusted)
+ untrusted = os.path.join(root, 'untrusted')
+ os.makedirs(untrusted)
self.roles.append(root)
return root
@@ -601,9 +606,9 @@
repo.delete_remote(repo.remotes.origin)
# is the playbook in a repo that we have already prepared?
- self.preparePlaybookRepos(args)
+ trusted, untrusted = self.preparePlaybookRepos(args)
- self.prepareRoles(args)
+ self.prepareRoles(args, trusted, untrusted)
# TODOv3: Ansible the ansible thing here.
self.prepareAnsibleFiles(args)
@@ -737,15 +742,24 @@
return None
def preparePlaybookRepos(self, args):
+ trusted = untrusted = False
for playbook in args['pre_playbooks']:
jobdir_playbook = self.jobdir.addPrePlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, required=True)
+ if playbook['trusted']:
+ trusted = True
+ else:
+ untrusted = True
for playbook in args['playbooks']:
jobdir_playbook = self.jobdir.addPlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, required=False)
+ if playbook['trusted']:
+ trusted = True
+ else:
+ untrusted = True
if jobdir_playbook.path is not None:
self.jobdir.playbook = jobdir_playbook
break
@@ -756,6 +770,11 @@
jobdir_playbook = self.jobdir.addPostPlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, required=True)
+ if playbook['trusted']:
+ trusted = True
+ else:
+ untrusted = True
+ return (trusted, untrusted)
def preparePlaybookRepo(self, jobdir_playbook, playbook, args, required):
self.log.debug("Prepare playbook repo for %s" % (playbook,))
@@ -799,11 +818,11 @@
required=required,
trusted=playbook['trusted'])
- def prepareRoles(self, args):
+ def prepareRoles(self, args, trusted, untrusted):
for role in args['roles']:
if role['type'] == 'zuul':
root = self.jobdir.addRole()
- self.prepareZuulRole(args, role, root)
+ self.prepareZuulRole(args, role, root, trusted, untrusted)
def findRole(self, path, trusted=False):
d = os.path.join(path, 'tasks')
@@ -826,17 +845,22 @@
self._blockPluginDirs(os.path.join(path, entry))
return path
- def prepareZuulRole(self, args, role, root):
+ def prepareZuulRole(self, args, role, root, trusted, untrusted):
self.log.debug("Prepare zuul role for %s" % (role,))
# Check out the role repo if needed
source = self.executor_server.connections.getSource(
role['connection'])
project = source.getProject(role['project'])
- role_repo = None
- if not role['trusted']:
- # This is a project repo, so it is safe to use the already
- # checked out version (from speculative merging) of the
- # role
+ untrusted_role_repo = None
+ trusted_role_repo = None
+ trusted_root = os.path.join(root, 'trusted')
+ untrusted_root = os.path.join(root, 'untrusted')
+ name = role['target_name']
+
+ if untrusted:
+ # There is at least one untrusted playbook. For that
+ # case, use the already checked out version (from
+ # speculative merging) of the role.
for i in args['items']:
if (i['connection'] == role['connection'] and
@@ -847,27 +871,70 @@
path = os.path.join(self.jobdir.src_root,
project.canonical_hostname,
project.name)
- link = os.path.join(root, role['name'])
+ # The name of the symlink is the requested name of
+ # the role (which may be the repo name or may be
+ # something else; this can come into play if this
+ # is a bare role).
+ link = os.path.join(untrusted_root, name)
+ link = os.path.realpath(link)
+ if not link.startswith(os.path.realpath(untrusted_root)):
+ raise Exception("Invalid role name %s", name)
os.symlink(path, link)
- role_repo = link
+ untrusted_role_repo = link
break
- # The role repo is either a config repo, or it isn't in
- # the stack of changes we are testing, so check out the branch
- # tip into a dedicated space.
-
- if not role_repo:
- merger = self.executor_server._getMerger(root)
+ if trusted or not untrusted_role_repo:
+ # There is at least one trusted playbook which will need a
+ # trusted checkout of the role, or the role did not appear
+ # in the dependency chain for the change (in which case,
+ # there is no existing untrusted checkout of it). Check
+ # out the branch tip into a dedicated space.
+ merger = self.executor_server._getMerger(trusted_root)
merger.checkoutBranch(role['connection'], project.name,
'master')
- role_repo = os.path.join(root, project.canonical_hostname,
- project.name)
+ orig_repo_path = os.path.join(trusted_root,
+ project.canonical_hostname,
+ project.name)
+ if name != project.name:
+ # The requested name of the role is not the same as
+ # the project name, so rename the git repo as the
+ # requested name. It is the only item in this
+ # directory, so we don't need to worry about
+ # collisions.
+ target = os.path.join(trusted_root,
+ project.canonical_hostname,
+ name)
+ target = os.path.realpath(target)
+ if not target.startswith(os.path.realpath(trusted_root)):
+ raise Exception("Invalid role name %s", name)
+ os.rename(orig_repo_path, target)
+ trusted_role_repo = target
+ else:
+ trusted_role_repo = orig_repo_path
- role_path = self.findRole(role_repo, trusted=role['trusted'])
- if role_path is None:
- # In the case of a bare role, add the containing directory
- role_path = os.path.join(root, project.canonical_hostname)
- self.jobdir.roles_path.append(role_path)
+ if not untrusted_role_repo:
+ # In the case that there was no untrusted checkout,
+ # use the trusted checkout.
+ untrusted_role_repo = trusted_role_repo
+ untrusted_root = trusted_root
+
+ if untrusted:
+ untrusted_role_path = self.findRole(untrusted_role_repo,
+ trusted=False)
+ if untrusted_role_path is None:
+ # In the case of a bare role, add the containing directory
+ untrusted_role_path = os.path.join(untrusted_root,
+ project.canonical_hostname)
+ self.jobdir.untrusted_roles_path.append(untrusted_role_path)
+
+ if trusted:
+ trusted_role_path = self.findRole(trusted_role_repo,
+ trusted=True)
+ if trusted_role_path is None:
+ # In the case of a bare role, add the containing directory
+ trusted_role_path = os.path.join(trusted_root,
+ project.canonical_hostname)
+ self.jobdir.trusted_roles_path.append(trusted_role_path)
def prepareAnsibleFiles(self, args):
keys = []
@@ -909,9 +976,6 @@
config.write('gathering = explicit\n')
config.write('library = %s\n'
% self.executor_server.library_dir)
- if self.jobdir.roles_path:
- config.write('roles_path = %s\n' %
- ':'.join(self.jobdir.roles_path))
config.write('command_warnings = False\n')
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
@@ -924,6 +988,12 @@
% self.executor_server.action_dir)
config.write('lookup_plugins = %s\n'
% self.executor_server.lookup_dir)
+ roles_path = self.jobdir.untrusted_roles_path
+ else:
+ roles_path = self.jobdir.trusted_roles_path
+
+ if roles_path:
+ config.write('roles_path = %s\n' % ':'.join(roles_path))
# On trusted jobs, we want to prevent the printing of args,
# since trusted jobs might have access to secrets that they may
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 403aca6..f5cce7b 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -18,6 +18,7 @@
import zuul.driver.zuul
import zuul.driver.gerrit
import zuul.driver.git
+import zuul.driver.github
import zuul.driver.smtp
import zuul.driver.timer
import zuul.driver.sql
@@ -40,6 +41,7 @@
self.registerDriver(zuul.driver.zuul.ZuulDriver())
self.registerDriver(zuul.driver.gerrit.GerritDriver())
self.registerDriver(zuul.driver.git.GitDriver())
+ self.registerDriver(zuul.driver.github.GithubDriver())
self.registerDriver(zuul.driver.smtp.SMTPDriver())
self.registerDriver(zuul.driver.timer.TimerDriver())
self.registerDriver(zuul.driver.sql.SQLDriver())
diff --git a/zuul/model.py b/zuul/model.py
index af80028..c2b4a9a 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -673,11 +673,10 @@
class ZuulRole(Role):
"""A reference to an ansible role in a Zuul project."""
- def __init__(self, target_name, connection_name, project_name, trusted):
+ def __init__(self, target_name, connection_name, project_name):
super(ZuulRole, self).__init__(target_name)
self.connection_name = connection_name
self.project_name = project_name
- self.trusted = trusted
def __repr__(self):
return '<ZuulRole %s %s>' % (self.project_name, self.target_name)
@@ -687,8 +686,7 @@
return False
return (super(ZuulRole, self).__eq__(other) and
self.connection_name == other.connection_name,
- self.project_name == other.project_name,
- self.trusted == other.trusted)
+ self.project_name == other.project_name)
def toDict(self):
# Render to a dict to use in passing json to the executor
@@ -696,7 +694,6 @@
d['type'] = 'zuul'
d['connection'] = self.connection_name
d['project'] = self.project_name
- d['trusted'] = self.trusted
return d
@@ -1822,7 +1819,7 @@
def getBasePath(self):
if hasattr(self, 'refspec'):
return "%s/%s/%s" % (
- self.number[-2:], self.number, self.patchset)
+ str(self.number)[-2:], self.number, self.patchset)
return super(Change, self).getBasePath()
def equals(self, other):
@@ -1859,6 +1856,20 @@
patchset=self.patchset)
+class PullRequest(Change):
+ def __init__(self, project):
+ super(PullRequest, self).__init__(project)
+ self.updated_at = None
+
+ def isUpdateOf(self, other):
+ if (hasattr(other, 'number') and self.number == other.number and
+ hasattr(other, 'patchset') and self.patchset != other.patchset and
+ hasattr(other, 'updated_at') and
+ self.updated_at > other.updated_at):
+ return True
+ return False
+
+
class TriggerEvent(object):
"""Incoming event from an external system."""
def __init__(self):
@@ -1880,6 +1891,8 @@
self.approvals = []
self.branch = None
self.comment = None
+ self.label = None
+ self.unlabel = None
# ref-updated
self.ref = None
self.oldrev = None
@@ -1910,6 +1923,25 @@
return ret
+ def isPatchsetCreated(self):
+ return 'patchset-created' == self.type
+
+ def isChangeAbandoned(self):
+ return 'change-abandoned' == self.type
+
+
+class GithubTriggerEvent(TriggerEvent):
+
+ def isPatchsetCreated(self):
+ if self.type == 'pull_request':
+ return self.action in ['opened', 'changed']
+ return False
+
+ def isChangeAbandoned(self):
+ if self.type == 'pull_request':
+ return 'closed' == self.action
+ return False
+
class BaseFilter(object):
"""Base Class for filtering which Changes and Events to process."""
@@ -2008,7 +2040,8 @@
def __init__(self, trigger, types=[], branches=[], refs=[],
event_approvals={}, comments=[], emails=[], usernames=[],
timespecs=[], required_approvals=[], reject_approvals=[],
- pipelines=[], ignore_deletes=True):
+ pipelines=[], actions=[], labels=[], unlabels=[],
+ ignore_deletes=True):
super(EventFilter, self).__init__(
required_approvals=required_approvals,
reject_approvals=reject_approvals)
@@ -2027,8 +2060,11 @@
self.emails = [re.compile(x) for x in emails]
self.usernames = [re.compile(x) for x in usernames]
self.pipelines = [re.compile(x) for x in pipelines]
+ self.actions = actions
self.event_approvals = event_approvals
self.timespecs = timespecs
+ self.labels = labels
+ self.unlabels = unlabels
self.ignore_deletes = ignore_deletes
def __repr__(self):
@@ -2061,6 +2097,12 @@
ret += ' username_filters: %s' % ', '.join(self._usernames)
if self.timespecs:
ret += ' timespecs: %s' % ', '.join(self.timespecs)
+ if self.actions:
+ ret += ' actions: %s' % ', '.join(self.actions)
+ if self.labels:
+ ret += ' labels: %s' % ', '.join(self.labels)
+ if self.unlabels:
+ ret += ' unlabels: %s' % ', '.join(self.unlabels)
ret += '>'
return ret
@@ -2157,6 +2199,22 @@
if self.timespecs and not matches_timespec:
return False
+ # actions are ORed
+ matches_action = False
+ for action in self.actions:
+ if (event.action == action):
+ matches_action = True
+ if self.actions and not matches_action:
+ return False
+
+ # labels are ORed
+ if self.labels and event.label not in self.labels:
+ return False
+
+ # unlabels are ORed
+ if self.unlabels and event.unlabel not in self.unlabels:
+ return False
+
return True
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 53ca4c1..2e9bef2 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -759,9 +759,9 @@
change.project.unparsed_config = None
self.reconfigureTenant(tenant)
for pipeline in tenant.layout.pipelines.values():
- if event.type == 'patchset-created':
+ if event.isPatchsetCreated():
pipeline.manager.removeOldVersionsOfChange(change)
- elif event.type == 'change-abandoned':
+ elif event.isChangeAbandoned():
pipeline.manager.removeAbandonedChange(change)
if pipeline.manager.eventMatches(event, change):
pipeline.manager.addChange(change)
diff --git a/zuul/webapp.py b/zuul/webapp.py
index f5a7373..37d6ddd 100644
--- a/zuul/webapp.py
+++ b/zuul/webapp.py
@@ -45,7 +45,7 @@
class WebApp(threading.Thread):
log = logging.getLogger("zuul.WebApp")
- change_path_regexp = '/status/change/(\d+,\d+)$'
+ change_path_regexp = '/status/change/(.*)$'
def __init__(self, scheduler, port=8001, cache_expiry=1,
listen_address='0.0.0.0'):