Add /info and /{tenant}/info route to zuul-web
There are a few pieces of information that are useful to know in the web
layer.
websocket_url is a config setting that, if set, is needed by the
console streaming. We currently pass this in appended to the streaming
url as a url parameter (which since it's a URL is a bit extra odd)
The endpoint is normally relative to the webapp,
but may need to be overridden in cases like publishing the html and
javascript to a disconnected location such as the draft output into the
log server in openstack or publishing built html/javascript to swift.
Add WebInfo and TenantWebInfo objects and corresponding /info and
/{tenant}/info routes. As an alternative, we could collapse WebInfo
and TenantWebInfo to just WebInfo and leave the tenant field set to None
for the /info route.
Some of the API functions are optionally provided by
plugins. The github plugin provides webhook URLs and the SQLReporter
plugin is needed for the builds endpoints. Add a Capabilities object
that can report on the existance of such things and pass it to plugin
route registration so that capabilities can be registered.
Add support for configuring stats_url
The old zuul status page had sparklines and other graphs on it, which
are not present in the current one because the graphite server wasn't
parameterized.
Add a config setting allowing a URL to a graphite server to be set and
expose that in the /info endpoint. Since statsd itself can emit to multiple
different backends, add a setting for the type of server, defaulting to
graphite.
Change-Id: I606a3b2cdf03cb73aa3ffd69d9d64c171b23b97a
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index ba14752..84ebc10 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -660,6 +660,16 @@
Base URL on which the websocket service is exposed, if different
than the base URL of the web app.
+ .. attr:: stats_url
+
+ Base URL from which statistics emitted via statsd can be queried.
+
+ .. attr:: stats_type
+ :default: graphite
+
+ Type of server hosting the statistics information. Currently only
+ 'graphite' is supported by the dashboard.
+
.. attr:: static_cache_expiry
:default: 3600
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index e150a47..602209f 100644
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -28,11 +28,24 @@
from tests.base import ZuulTestCase, FIXTURE_DIR
-class TestWeb(ZuulTestCase):
+class FakeConfig(object):
+
+ def __init__(self, config):
+ self.config = config or {}
+
+ def has_option(self, section, option):
+ return option in self.config.get(section, {})
+
+ def get(self, section, option):
+ return self.config.get(section, {}).get(option)
+
+
+class BaseTestWeb(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
+ config_ini_data = {}
def setUp(self):
- super(TestWeb, self).setUp()
+ super(BaseTestWeb, self).setUp()
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('Code-Review', 2)
@@ -42,10 +55,13 @@
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
+ self.zuul_ini_config = FakeConfig(self.config_ini_data)
# Start the web server
self.web = zuul.web.ZuulWeb(
listen_address='127.0.0.1', listen_port=0,
- gear_server='127.0.0.1', gear_port=self.gearman_server.port)
+ gear_server='127.0.0.1', gear_port=self.gearman_server.port,
+ info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config)
+ )
loop = asyncio.new_event_loop()
loop.set_debug(True)
ws_thread = threading.Thread(target=self.web.run, args=(loop,))
@@ -72,7 +88,10 @@
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
- super(TestWeb, self).tearDown()
+ super(BaseTestWeb, self).tearDown()
+
+
+class TestWeb(BaseTestWeb):
def test_web_status(self):
"Test that we can retrieve JSON status info"
@@ -215,3 +234,78 @@
e = self.assertRaises(
urllib.error.HTTPError, urllib.request.urlopen, req)
self.assertEqual(404, e.code)
+
+
+class TestInfo(BaseTestWeb):
+
+ def setUp(self):
+ super(TestInfo, self).setUp()
+ web_config = self.config_ini_data.get('web', {})
+ self.websocket_url = web_config.get('websocket_url')
+ self.stats_url = web_config.get('stats_url')
+ statsd_config = self.config_ini_data.get('statsd', {})
+ self.stats_prefix = statsd_config.get('prefix')
+
+ def test_info(self):
+ req = urllib.request.Request(
+ "http://localhost:%s/info" % self.port)
+ f = urllib.request.urlopen(req)
+ info = json.loads(f.read().decode('utf8'))
+ self.assertEqual(
+ info, {
+ "info": {
+ "endpoint": "http://localhost:%s" % self.port,
+ "capabilities": {
+ "job_history": False
+ },
+ "stats": {
+ "url": self.stats_url,
+ "prefix": self.stats_prefix,
+ "type": "graphite",
+ },
+ "websocket_url": self.websocket_url,
+ }
+ })
+
+ def test_tenant_info(self):
+ req = urllib.request.Request(
+ "http://localhost:%s/tenant-one/info" % self.port)
+ f = urllib.request.urlopen(req)
+ info = json.loads(f.read().decode('utf8'))
+ self.assertEqual(
+ info, {
+ "info": {
+ "endpoint": "http://localhost:%s" % self.port,
+ "tenant": "tenant-one",
+ "capabilities": {
+ "job_history": False
+ },
+ "stats": {
+ "url": self.stats_url,
+ "prefix": self.stats_prefix,
+ "type": "graphite",
+ },
+ "websocket_url": self.websocket_url,
+ }
+ })
+
+
+class TestWebSocketInfo(TestInfo):
+
+ config_ini_data = {
+ 'web': {
+ 'websocket_url': 'wss://ws.example.com'
+ }
+ }
+
+
+class TestGraphiteUrl(TestInfo):
+
+ config_ini_data = {
+ 'statsd': {
+ 'prefix': 'example'
+ },
+ 'web': {
+ 'stats_url': 'https://graphite.example.com',
+ }
+ }
diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py
index abdb1cb..8b0e3ee 100755
--- a/zuul/cmd/web.py
+++ b/zuul/cmd/web.py
@@ -20,6 +20,7 @@
import threading
import zuul.cmd
+import zuul.model
import zuul.web
from zuul.lib.config import get_default
@@ -33,8 +34,11 @@
self.web.stop()
def _run(self):
+ info = zuul.model.WebInfo.fromConfig(self.config)
+
params = dict()
+ params['info'] = info
params['listen_address'] = get_default(self.config,
'web', 'listen_address',
'127.0.0.1')
diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py
index 86f14d6..1c62f4d 100644
--- a/zuul/connection/__init__.py
+++ b/zuul/connection/__init__.py
@@ -75,11 +75,14 @@
still in use. Anything in our cache that isn't in the supplied
list should be safe to remove from the cache."""
- def getWebHandlers(self, zuul_web):
+ def getWebHandlers(self, zuul_web, info):
"""Return a list of web handlers to register with zuul-web.
:param zuul.web.ZuulWeb zuul_web:
Zuul Web instance.
+ :param zuul.model.WebInfo info:
+ The WebInfo object for the Zuul Web instance. Can be used by
+ plugins to toggle API capabilities.
:returns: List of `zuul.web.handler.BaseWebHandler` instances.
"""
return []
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 6dfcdd3..772ba9b 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -1141,7 +1141,7 @@
return statuses
- def getWebHandlers(self, zuul_web):
+ def getWebHandlers(self, zuul_web, info):
return [GithubWebhookHandler(self, zuul_web, 'POST', 'payload')]
def validateWebConfig(self, config, connections):
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index d16a623..e931301 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -125,7 +125,8 @@
return zuul_buildset_table, zuul_build_table
- def getWebHandlers(self, zuul_web):
+ def getWebHandlers(self, zuul_web, info):
+ info.capabilities.job_history = True
return [
SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'),
StaticHandler(zuul_web, '/{tenant}/builds.html'),
diff --git a/zuul/model.py b/zuul/model.py
index bd4f4d1..44e8d06 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -24,6 +24,7 @@
import textwrap
from zuul import change_matcher
+from zuul.lib.config import get_default
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
@@ -3182,3 +3183,80 @@
td = self._getTD(build)
td.add(elapsed, result)
td.save()
+
+
+class Capabilities(object):
+ """The set of capabilities this Zuul installation has.
+
+ Some plugins add elements to the external API. In order to
+ facilitate consumers knowing if functionality is available
+ or not, keep track of distinct capability flags.
+ """
+ def __init__(self, job_history=False):
+ self.job_history = job_history
+
+ def __repr__(self):
+ return '<Capabilities 0x%x %s>' % (id(self), self._renderFlags())
+
+ def _renderFlags(self):
+ d = self.toDict()
+ return " ".join(['{k}={v}'.format(k=k, v=v) for (k, v) in d.items()])
+
+ def copy(self):
+ return Capabilities(**self.toDict())
+
+ def toDict(self):
+ d = dict()
+ d['job_history'] = self.job_history
+ return d
+
+
+class WebInfo(object):
+ """Information about the system needed by zuul-web /info."""
+
+ def __init__(self, websocket_url=None, endpoint=None,
+ capabilities=None, stats_url=None,
+ stats_prefix=None, stats_type=None):
+ self.capabilities = capabilities or Capabilities()
+ self.websocket_url = websocket_url
+ self.stats_url = stats_url
+ self.stats_prefix = stats_prefix
+ self.stats_type = stats_type
+ self.endpoint = endpoint
+ self.tenant = None
+
+ def __repr__(self):
+ return '<WebInfo 0x%x capabilities=%s>' % (
+ id(self), str(self.capabilities))
+
+ def copy(self):
+ return WebInfo(
+ websocket_url=self.websocket_url,
+ endpoint=self.endpoint,
+ stats_url=self.stats_url,
+ stats_prefix=self.stats_prefix,
+ stats_type=self.stats_type,
+ capabilities=self.capabilities.copy())
+
+ @staticmethod
+ def fromConfig(config):
+ return WebInfo(
+ websocket_url=get_default(config, 'web', 'websocket_url', None),
+ stats_url=get_default(config, 'web', 'stats_url', None),
+ stats_prefix=get_default(config, 'statsd', 'prefix'),
+ stats_type=get_default(config, 'web', 'stats_type', 'graphite'),
+ )
+
+ def toDict(self):
+ d = dict()
+ d['websocket_url'] = self.websocket_url
+ stats = dict()
+ stats['url'] = self.stats_url
+ stats['prefix'] = self.stats_prefix
+ stats['type'] = self.stats_type
+ d['stats'] = stats
+ d['endpoint'] = self.endpoint
+ d['capabilities'] = self.capabilities.toDict()
+ if self.tenant:
+ d['tenant'] = self.tenant
+ return d
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index 41b1b81..7a1af30 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -26,6 +26,7 @@
import aiohttp
from aiohttp import web
+import zuul.model
import zuul.rpcclient
from zuul.web.handler import StaticHandler
@@ -235,13 +236,16 @@
gear_server, gear_port,
ssl_key=None, ssl_cert=None, ssl_ca=None,
static_cache_expiry=3600,
- connections=None):
+ connections=None,
+ info=None):
+ self.start_time = time.time()
self.listen_address = listen_address
self.listen_port = listen_port
self.event_loop = None
self.term = None
self.server = None
self.static_cache_expiry = static_cache_expiry
+ self.info = info
# instanciate handlers
self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
ssl_key, ssl_cert, ssl_ca)
@@ -250,12 +254,37 @@
self._plugin_routes = [] # type: List[zuul.web.handler.BaseWebHandler]
connections = connections or []
for connection in connections:
- self._plugin_routes.extend(connection.getWebHandlers(self))
+ self._plugin_routes.extend(
+ connection.getWebHandlers(self, self.info))
async def _handleWebsocket(self, request):
return await self.log_streaming_handler.processRequest(
request)
+ async def _handleRootInfo(self, request):
+ info = self.info.copy()
+ info.endpoint = str(request.url.parent)
+ return self._handleInfo(info)
+
+ def _handleTenantInfo(self, request):
+ info = self.info.copy()
+ info.tenant = request.match_info["tenant"]
+ # yarl.URL.parent on a root url returns the root url, so this is
+ # both safe and accurate for white-labeled tenants like OpenStack,
+ # zuul-web running on / and zuul-web running on a sub-url like
+ # softwarefactory-project.io
+ info.endpoint = str(request.url.parent.parent.parent)
+ return self._handleInfo(info)
+
+ def _handleInfo(self, info):
+ resp = web.json_response({'info': info.toDict()}, status=200)
+ resp.headers['Access-Control-Allow-Origin'] = '*'
+ if self.static_cache_expiry:
+ resp.headers['Cache-Control'] = "public, max-age=%d" % \
+ self.static_cache_expiry
+ resp.last_modified = self.start_time
+ return resp
+
async def _handleTenantsRequest(self, request):
return await self.gearman_handler.processRequest(request,
'tenant_list')
@@ -286,6 +315,8 @@
is run within a separate (non-main) thread.
"""
routes = [
+ ('GET', '/info', self._handleRootInfo),
+ ('GET', '/{tenant}/info', self._handleTenantInfo),
('GET', '/tenants', self._handleTenantsRequest),
('GET', '/{tenant}/status', self._handleStatusRequest),
('GET', '/{tenant}/jobs', self._handleJobsRequest),