Move github webhook from webapp to zuul-web
We want to have zuul-web to handle all http serving stuff so also the
github webhook handling needs to be moved to zuul-web.
Note that this changes the url of the github webhooks to
/driver/github/<connection_name>/payload.
Change-Id: I6482de6c5b9655ac0b9bf353b37a59cd5406f1b7
Signed-off-by: Jesse Keating <omgjlk@us.ibm.com>
Co-Authored-by: Tobias Henkel <tobias.henkel@bmw.de>
diff --git a/tests/base.py b/tests/base.py
index c449242..e7151df 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -66,9 +66,11 @@
import zuul.merger.server
import zuul.model
import zuul.nodepool
+import zuul.rpcclient
import zuul.zk
import zuul.configloader
from zuul.exceptions import MergeFailure
+from zuul.lib.config import get_default
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'fixtures')
@@ -939,7 +941,7 @@
class FakeGithubConnection(githubconnection.GithubConnection):
log = logging.getLogger("zuul.test.FakeGithubConnection")
- def __init__(self, driver, connection_name, connection_config,
+ def __init__(self, driver, connection_name, connection_config, rpcclient,
changes_db=None, upstream_root=None):
super(FakeGithubConnection, self).__init__(driver, connection_name,
connection_config)
@@ -952,12 +954,16 @@
self.merge_not_allowed_count = 0
self.reports = []
self.github_client = tests.fakegithub.FakeGithub(changes_db)
+ self.rpcclient = rpcclient
def getGithubClient(self,
project=None,
user_id=None):
return self.github_client
+ def setZuulWebPort(self, port):
+ self.zuul_web_port = port
+
def openFakePullRequest(self, project, branch, subject, files=[],
body=None):
self.pr_number += 1
@@ -991,19 +997,25 @@
}
return (name, data)
- def emitEvent(self, event):
+ def emitEvent(self, event, use_zuulweb=False):
"""Emulates sending the GitHub webhook event to the connection."""
- port = self.webapp.server.socket.getsockname()[1]
name, data = event
payload = json.dumps(data).encode('utf8')
secret = self.connection_config['webhook_token']
signature = githubconnection._sign_request(payload, secret)
- headers = {'X-Github-Event': name, 'X-Hub-Signature': signature}
- req = urllib.request.Request(
- 'http://localhost:%s/connection/%s/payload'
- % (port, self.connection_name),
- data=payload, headers=headers)
- return urllib.request.urlopen(req)
+ headers = {'x-github-event': name, 'x-hub-signature': signature}
+
+ if use_zuulweb:
+ req = urllib.request.Request(
+ 'http://127.0.0.1:%s/driver/github/%s/payload'
+ % (self.zuul_web_port, self.connection_name),
+ data=payload, headers=headers)
+ return urllib.request.urlopen(req)
+ else:
+ job = self.rpcclient.submitJob(
+ 'github:%s:payload' % self.connection_name,
+ {'headers': headers, 'body': data})
+ return json.loads(job.data[0])
def addProject(self, project):
# use the original method here and additionally register it in the
@@ -1983,6 +1995,13 @@
'gearman', 'ssl_key',
os.path.join(FIXTURE_DIR, 'gearman/client.key'))
+ self.rpcclient = zuul.rpcclient.RPCClient(
+ self.config.get('gearman', 'server'),
+ self.gearman_server.port,
+ get_default(self.config, 'gearman', 'ssl_key'),
+ get_default(self.config, 'gearman', 'ssl_cert'),
+ get_default(self.config, 'gearman', 'ssl_ca'))
+
gerritsource.GerritSource.replication_timeout = 1.5
gerritsource.GerritSource.replication_retry_interval = 0.5
gerritconnection.GerritEventConnector.delay = 0.0
@@ -2000,7 +2019,7 @@
]
self.configure_connections()
- self.sched.registerConnections(self.connections, self.webapp)
+ self.sched.registerConnections(self.connections)
self.executor_server = RecordingExecutorServer(
self.config, self.connections,
@@ -2065,6 +2084,7 @@
server = config.get('server', 'github.com')
db = self.github_changes_dbs.setdefault(server, {})
con = FakeGithubConnection(driver, name, config,
+ self.rpcclient,
changes_db=db,
upstream_root=self.upstream_root)
self.event_queues.append(con.event_queue)
@@ -2297,6 +2317,7 @@
self.statsd.join()
self.webapp.stop()
self.webapp.join()
+ self.rpcclient.shutdown()
self.gearman_server.shutdown()
self.fake_nodepool.stop()
self.zk.disconnect()
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 3942b0b..7aca428 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -12,15 +12,20 @@
# License for the specific language governing permissions and limitations
# under the License.
+import asyncio
+import threading
import os
import re
from testtools.matchers import MatchesRegex, StartsWith
import urllib
+import socket
import time
from unittest import skip
import git
+import zuul.web
+
from tests.base import ZuulTestCase, simple_layout, random_sha1
@@ -734,3 +739,85 @@
# project2 should have no parsed branch
self.assertEqual(0, len(project2.unparsed_branch_config.keys()))
+
+
+class TestGithubWebhook(ZuulTestCase):
+ config_file = 'zuul-github-driver.conf'
+
+ def setUp(self):
+ super(TestGithubWebhook, self).setUp()
+
+ # Start the web server
+ self.web = zuul.web.ZuulWeb(
+ listen_address='127.0.0.1', listen_port=0,
+ gear_server='127.0.0.1', gear_port=self.gearman_server.port,
+ github_connections={'github': self.fake_github})
+ loop = asyncio.new_event_loop()
+ loop.set_debug(True)
+ ws_thread = threading.Thread(target=self.web.run, args=(loop,))
+ ws_thread.start()
+ self.addCleanup(loop.close)
+ self.addCleanup(ws_thread.join)
+ self.addCleanup(self.web.stop)
+
+ host = '127.0.0.1'
+ # Wait until web server is started
+ while True:
+ time.sleep(0.1)
+ if self.web.server is None:
+ continue
+ port = self.web.server.sockets[0].getsockname()[1]
+ try:
+ with socket.create_connection((host, port)):
+ break
+ except ConnectionRefusedError:
+ pass
+
+ self.fake_github.setZuulWebPort(port)
+
+ def tearDown(self):
+ super(TestGithubWebhook, self).tearDown()
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_webhook(self):
+ """Test that we can get github events via zuul-web."""
+
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ self.fake_github.emitEvent(A.getPullRequestOpenedEvent(),
+ use_zuulweb=True)
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-test1').result)
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('project-test2').result)
+
+ job = self.getJobFromHistory('project-test2')
+ zuulvars = job.parameters['zuul']
+ self.assertEqual(str(A.number), zuulvars['change'])
+ self.assertEqual(str(A.head_sha), zuulvars['patchset'])
+ self.assertEqual('master', zuulvars['branch'])
+ self.assertEqual(1, len(A.comments))
+ self.assertThat(
+ A.comments[0],
+ MatchesRegex('.*\[project-test1 \]\(.*\).*', re.DOTALL))
+ self.assertThat(
+ A.comments[0],
+ MatchesRegex('.*\[project-test2 \]\(.*\).*', re.DOTALL))
+ self.assertEqual(2, len(self.history))
+
+ # test_pull_unmatched_branch_event(self):
+ self.create_branch('org/project', 'unmatched_branch')
+ B = self.fake_github.openFakePullRequest(
+ 'org/project', 'unmatched_branch', 'B')
+ self.fake_github.emitEvent(B.getPullRequestOpenedEvent(),
+ use_zuulweb=True)
+ self.waitUntilSettled()
+
+ self.assertEqual(2, len(self.history))