Merge "Emit phase banner separate from play banner" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index a0de922..cc9d181 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -385,6 +385,10 @@
port=9000
+**websocket_url**
+ Base URL on which the websocket service is exposed, if different than the
+ base URL of the web app.
+
Operation
~~~~~~~~~
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 76f73c3..c137918 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -34,13 +34,12 @@
When Zuul starts, it examines all of the git repositories which are
specified by the system administrator in :ref:`tenant-config` and searches
-for files in the root of each repository. In the case of a
-*config-project*, Zuul looks for a file named `zuul.yaml`. In the
-case of an *untrusted-project*, Zuul looks first for `zuul.yaml` and
-if that is not found, `.zuul.yaml` (with a leading dot). In the case
-of an *untrusted-project*, the configuration from every branch is
-included, however, in the case of a *config-project*, only the
-`master` branch is examined.
+for files in the root of each repository. Zuul looks first for a file named
+`zuul.yaml` or a directory named `zuul.d`, and if they are not found,
+`.zuul.yaml` or `.zuul.d` (with a leading dot). In the case of an
+*untrusted-project*, the configuration from every branch is included,
+however, in the case of a *config-project*, only the `master` branch is
+examined.
When a change is proposed to one of these files in an
*untrusted-project*, the configuration proposed in the change is
@@ -64,6 +63,16 @@
YAML-formatted and are structured as a series of items, each of which
is described below.
+In the case of a `zuul.d` directory, Zuul recurses the directory and extends
+the configuration using all the .yaml files in the sorted path order.
+For example, to keep job's variants in a separate file, it needs to be loaded
+after the main entries, for example using number prefixes in file's names::
+
+* zuul.d/pipelines.yaml
+* zuul.d/projects.yaml
+* zuul.d/01_jobs.yaml
+* zuul.d/02_jobs-variants.yaml
+
.. _pipeline:
Pipeline
@@ -738,6 +747,12 @@
be installed (and therefore referenced from Ansible), the `name`
attribute may be used to specify an alternate.
+ A job automatically has the project in which it is defined added to
+ the roles path if that project appears to contain a role or `roles/`
+ directory. By default, the project is added to the path under its
+ own name, however, that may be changed by explicitly listing the
+ project in the roles list in the usual way.
+
.. note:: galaxy roles are not yet implemented
**galaxy**
diff --git a/tests/fixtures/config/conflict-config/git/common-config/.zuul.d/jobs.yaml b/tests/fixtures/config/conflict-config/git/common-config/.zuul.d/jobs.yaml
new file mode 100644
index 0000000..20056ee
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/git/common-config/.zuul.d/jobs.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: trusted-.zuul.d-jobs
diff --git a/tests/fixtures/config/conflict-config/git/common-config/.zuul.yaml b/tests/fixtures/config/conflict-config/git/common-config/.zuul.yaml
new file mode 100644
index 0000000..da2bc1e
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/git/common-config/.zuul.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: trusted-.zuul.yaml-job
diff --git a/tests/fixtures/config/conflict-config/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/conflict-config/git/common-config/zuul.d/jobs.yaml
new file mode 100644
index 0000000..5a92f43
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/git/common-config/zuul.d/jobs.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: trusted-zuul.d-jobs
diff --git a/tests/fixtures/config/conflict-config/git/common-config/zuul.yaml b/tests/fixtures/config/conflict-config/git/common-config/zuul.yaml
new file mode 100644
index 0000000..792fc8f
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/git/common-config/zuul.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: trusted-zuul.yaml-job
diff --git a/tests/fixtures/config/conflict-config/git/org_project/.zuul.yaml b/tests/fixtures/config/conflict-config/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..dc1ff45
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/git/org_project/.zuul.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: untrusted-.zuul.yaml-job
diff --git a/tests/fixtures/config/conflict-config/git/org_project/zuul.yaml b/tests/fixtures/config/conflict-config/git/org_project/zuul.yaml
new file mode 100644
index 0000000..cc63564
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/git/org_project/zuul.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: untrusted-zuul.yaml-job
diff --git a/tests/fixtures/config/conflict-config/main.yaml b/tests/fixtures/config/conflict-config/main.yaml
new file mode 100644
index 0000000..208e274
--- /dev/null
+++ b/tests/fixtures/config/conflict-config/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
diff --git a/tests/fixtures/config/implicit-roles/git/common-config/zuul.yaml b/tests/fixtures/config/implicit-roles/git/common-config/zuul.yaml
new file mode 100644
index 0000000..ba91fb5
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/common-config/zuul.yaml
@@ -0,0 +1,12 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
new file mode 100644
index 0000000..74c8e8e
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
@@ -0,0 +1,15 @@
+- job:
+ name: implicit-role-fail
+
+- job:
+ name: explicit-role-fail
+ attempts: 1
+ roles:
+ - zuul: org/norole-project
+
+- project:
+ name: org/norole-project
+ check:
+ jobs:
+ - implicit-role-fail
+ - explicit-role-fail
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/README b/tests/fixtures/config/implicit-roles/git/org_norole-project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/explicit-role-fail.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/explicit-role-fail.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/explicit-role-fail.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/implicit-role-fail.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/implicit-role-fail.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/implicit-role-fail.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
new file mode 100644
index 0000000..42cae95
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
@@ -0,0 +1,15 @@
+- job:
+ name: implicit-role-ok
+
+- job:
+ name: explicit-role-ok
+ roles:
+ - zuul: org/role-project
+ name: role-name
+
+- project:
+ name: org/role-project
+ check:
+ jobs:
+ - implicit-role-ok
+ - explicit-role-ok
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/README b/tests/fixtures/config/implicit-roles/git/org_role-project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/explicit-role-ok.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/explicit-role-ok.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/explicit-role-ok.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/implicit-role-ok.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/implicit-role-ok.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/implicit-role-ok.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/roles/README b/tests/fixtures/config/implicit-roles/git/org_role-project/roles/README
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/roles/README
diff --git a/tests/fixtures/config/implicit-roles/main.yaml b/tests/fixtures/config/implicit-roles/main.yaml
new file mode 100644
index 0000000..d5e481a
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/norole-project
+ - org/role-project
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
index 573ccbf..d08c6a1 100644
--- a/tests/unit/test_configloader.py
+++ b/tests/unit/test_configloader.py
@@ -12,6 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
+import fixtures
+import logging
import textwrap
from tests.base import ZuulTestCase
@@ -245,3 +247,39 @@
# project1-project2-integration test removed, only want project-test1
self.assertHistory([
dict(name='project-test1', result='SUCCESS', changes='1,1')])
+
+ def test_config_path_conflict(self):
+ def add_file(project, path):
+ new_file = textwrap.dedent(
+ """
+ - job:
+ name: test-job
+ """
+ )
+ file_dict = {path: new_file}
+ A = self.fake_gerrit.addFakeChange(project, 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ log_fixture = self.useFixture(
+ fixtures.FakeLogger(level=logging.WARNING))
+
+ log_fixture._output.truncate(0)
+ add_file("common-config", "zuul.yaml")
+ self.assertIn("Multiple configuration", log_fixture.output)
+
+ log_fixture._output.truncate(0)
+ add_file("org/project1", ".zuul.yaml")
+ self.assertIn("Multiple configuration", log_fixture.output)
+
+
+class TestConfigConflict(ZuulTestCase):
+ tenant_config_file = 'config/conflict-config/main.yaml'
+
+ def test_conflict_config(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ jobs = sorted(tenant.layout.jobs.keys())
+ self.assertEquals(
+ ['noop', 'trusted-zuul.yaml-job', 'untrusted-zuul.yaml-job'],
+ jobs)
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index ac2a779..d9cf839 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2290,10 +2290,14 @@
self.assertEqual('project-merge', status_jobs[0]['name'])
# TODO(mordred) pull uuids from self.builds
self.assertEqual(
+ 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
+ uuid=status_jobs[0]['uuid']),
+ status_jobs[0]['url'])
+ self.assertEqual(
'finger://{hostname}/{uuid}'.format(
hostname=self.executor_server.hostname,
uuid=status_jobs[0]['uuid']),
- status_jobs[0]['url'])
+ status_jobs[0]['finger_url'])
# TOOD(mordred) configure a success-url on the base job
self.assertEqual(
'finger://{hostname}/{uuid}'.format(
@@ -2302,10 +2306,14 @@
status_jobs[0]['report_url'])
self.assertEqual('project-test1', status_jobs[1]['name'])
self.assertEqual(
+ 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
+ uuid=status_jobs[1]['uuid']),
+ status_jobs[1]['url'])
+ self.assertEqual(
'finger://{hostname}/{uuid}'.format(
hostname=self.executor_server.hostname,
uuid=status_jobs[1]['uuid']),
- status_jobs[1]['url'])
+ status_jobs[1]['finger_url'])
self.assertEqual(
'finger://{hostname}/{uuid}'.format(
hostname=self.executor_server.hostname,
@@ -2314,10 +2322,14 @@
self.assertEqual('project-test2', status_jobs[2]['name'])
self.assertEqual(
+ 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
+ uuid=status_jobs[2]['uuid']),
+ status_jobs[2]['url'])
+ self.assertEqual(
'finger://{hostname}/{uuid}'.format(
hostname=self.executor_server.hostname,
uuid=status_jobs[2]['uuid']),
- status_jobs[2]['url'])
+ status_jobs[2]['finger_url'])
self.assertEqual(
'finger://{hostname}/{uuid}'.format(
hostname=self.executor_server.hostname,
@@ -3607,10 +3619,13 @@
self.assertEqual('gate', job['pipeline'])
self.assertEqual(False, job['retry'])
self.assertEqual(
+ 'static/stream.html?uuid={uuid}&logfile=console.log'
+ .format(uuid=job['uuid']), job['url'])
+ self.assertEqual(
'finger://{hostname}/{uuid}'.format(
hostname=self.executor_server.hostname,
uuid=job['uuid']),
- job['url'])
+ job['finger_url'])
self.assertEqual(2, len(job['worker']))
self.assertEqual(False, job['canceled'])
self.assertEqual(True, job['voting'])
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 734c45c..b162469 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -716,9 +716,7 @@
self.assertEqual(4096, private_key.key_size)
-class TestRoles(ZuulTestCase):
- tenant_config_file = 'config/roles/main.yaml'
-
+class RoleTestCase(ZuulTestCase):
def _assertRolePath(self, build, playbook, content):
path = os.path.join(self.test_root, build.uuid,
'ansible', playbook, 'ansible.cfg')
@@ -738,6 +736,10 @@
"Should have no roles_path line in %s" %
(playbook,))
+
+class TestRoles(RoleTestCase):
+ tenant_config_file = 'config/roles/main.yaml'
+
def test_role(self):
# This exercises a proposed change to a role being checked out
# and used.
@@ -822,6 +824,57 @@
A.messages[-1])
+class TestImplicitRoles(RoleTestCase):
+ tenant_config_file = 'config/implicit-roles/main.yaml'
+
+ def test_missing_roles(self):
+ # Test implicit and explicit roles for a project which does
+ # not have roles. The implicit role should be silently
+ # ignored since the project doesn't supply roles, but if a
+ # user declares an explicit role, it should error.
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/norole-project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 2)
+ build = self.getBuildByName('implicit-role-fail')
+ self._assertRolePath(build, 'playbook_0', None)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ # The retry_limit doesn't get recorded
+ self.assertHistory([
+ dict(name='implicit-role-fail', result='SUCCESS', changes='1,1'),
+ ])
+
+ def test_roles(self):
+ # Test implicit and explicit roles for a project which does
+ # have roles. In both cases, we should end up with the role
+ # in the path. In the explicit case, ensure we end up with
+ # the name we specified.
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/role-project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 2)
+ build = self.getBuildByName('implicit-role-ok')
+ self._assertRolePath(build, 'playbook_0', 'role_0')
+
+ build = self.getBuildByName('explicit-role-ok')
+ self._assertRolePath(build, 'playbook_0', 'role_0')
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='implicit-role-ok', result='SUCCESS', changes='1,1'),
+ dict(name='explicit-role-ok', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index dec15e7..b55aed8 100755
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -91,6 +91,7 @@
cmd_show = subparsers.add_parser('show',
help='valid show subcommands')
+ cmd_show.set_defaults(func=self.show_running_jobs)
show_subparsers = cmd_show.add_subparsers(title='show')
show_running_jobs = show_subparsers.add_parser(
'running-jobs',
@@ -108,6 +109,9 @@
show_running_jobs.set_defaults(func=self.show_running_jobs)
self.args = parser.parse_args()
+ if not getattr(self.args, 'func', None):
+ parser.print_help()
+ sys.exit(1)
if self.args.func == self.enqueue_ref:
if self.args.oldrev == self.args.newrev:
parser.error("The old and new revisions must not be the same.")
diff --git a/zuul/configloader.py b/zuul/configloader.py
index f8e2d15..7640dfc 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -17,6 +17,7 @@
import logging
import textwrap
import io
+import re
import voluptuous as vs
@@ -300,6 +301,8 @@
class JobParser(object):
+ ANSIBLE_ROLE_RE = re.compile(r'^(ansible[-_.+]*)*(role[-_.+]*)*')
+
@staticmethod
def getSchema():
auth = {'secrets': to_list(str),
@@ -425,14 +428,19 @@
# Roles are part of the playbook context so we must establish
# them earlier than playbooks.
+ roles = []
if 'roles' in conf:
- roles = []
for role in conf.get('roles', []):
if 'zuul' in role:
r = JobParser._makeZuulRole(tenant, job, role)
if r:
roles.append(r)
- job.addRoles(roles)
+ # A job's repo should be an implicit role source for that job,
+ # but not in a project-pipeline variant.
+ if not project_pipeline:
+ r = JobParser._makeImplicitRole(job)
+ roles.insert(0, r)
+ job.addRoles(roles)
for pre_run_name in as_list(conf.get('pre-run')):
pre_run = model.PlaybookContext(job.source_context,
@@ -554,6 +562,16 @@
project.connection_name,
project.name)
+ @staticmethod
+ def _makeImplicitRole(job):
+ project = job.source_context.project
+ name = project.name.split('/')[-1]
+ name = JobParser.ANSIBLE_ROLE_RE.sub('', name)
+ return model.ZuulRole(name,
+ project.connection_name,
+ project.name,
+ implicit=True)
+
class ProjectTemplateParser(object):
log = logging.getLogger("zuul.ProjectTemplateParser")
@@ -1213,7 +1231,7 @@
(job, job.files))
loaded = False
files = sorted(job.files.keys())
- for conf_root in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']:
+ for conf_root in ['zuul.yaml', 'zuul.d', '.zuul.yaml', '.zuul.d']:
for fn in files:
fn_root = fn.split('/')[0]
if fn_root != conf_root or not job.files.get(fn):
@@ -1416,8 +1434,7 @@
fns1.append(fn)
if fn.startswith(".zuul.d/"):
fns2.append(fn)
-
- fns = ['zuul.yaml', '.zuul.yaml'] + sorted(fns1) + sorted(fns2)
+ fns = ["zuul.yaml"] + sorted(fns1) + [".zuul.yaml"] + sorted(fns2)
incdata = None
loaded = None
for fn in fns:
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index f291dce..ae36263 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -51,6 +51,10 @@
pass
+class RoleNotFoundError(ExecutorError):
+ pass
+
+
class Watchdog(object):
def __init__(self, timeout, function, args):
self.timeout = timeout
@@ -1137,12 +1141,14 @@
if os.path.isdir(d):
# This repo has a collection of roles
if not trusted:
+ self._blockPluginDirs(d)
for entry in os.listdir(d):
- if os.path.isdir(os.path.join(d, entry)):
- self._blockPluginDirs(os.path.join(d, entry))
+ entry_path = os.path.join(d, entry)
+ if os.path.isdir(entry_path):
+ self._blockPluginDirs(entry_path)
return d
# It is neither a bare role, nor a collection of roles
- raise ExecutorError("Unable to find role in %s" % (path,))
+ raise RoleNotFoundError("Unable to find role in %s" % (path,))
def prepareZuulRole(self, jobdir_playbook, role, args, root):
self.log.debug("Prepare zuul role for %s" % (role,))
@@ -1183,10 +1189,17 @@
raise ExecutorError("Invalid role name %s", name)
os.symlink(path, link)
- role_path = self.findRole(link, trusted=jobdir_playbook.trusted)
+ try:
+ role_path = self.findRole(link, trusted=jobdir_playbook.trusted)
+ except RoleNotFoundError:
+ if role['implicit']:
+ self.log.info("Implicit role not found in %s", link)
+ return
+ raise
if role_path is None:
# In the case of a bare role, add the containing directory
role_path = root
+ self.log.debug("Adding role path %s", role_path)
jobdir_playbook.roles_path.append(role_path)
def prepareAnsibleFiles(self, args):
diff --git a/zuul/model.py b/zuul/model.py
index 1df70db..bc9eeb7 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -164,7 +164,7 @@
items.extend(shared_queue.queue)
return items
- def formatStatusJSON(self):
+ def formatStatusJSON(self, websocket_url=None):
j_pipeline = dict(name=self.name,
description=self.description)
j_queues = []
@@ -181,7 +181,7 @@
if j_changes:
j_queue['heads'].append(j_changes)
j_changes = []
- j_changes.append(e.formatJSON())
+ j_changes.append(e.formatJSON(websocket_url))
if (len(j_changes) > 1 and
(j_changes[-2]['remaining_time'] is not None) and
(j_changes[-1]['remaining_time'] is not None)):
@@ -702,10 +702,12 @@
class ZuulRole(Role):
"""A reference to an ansible role in a Zuul project."""
- def __init__(self, target_name, connection_name, project_name):
+ def __init__(self, target_name, connection_name, project_name,
+ implicit=False):
super(ZuulRole, self).__init__(target_name)
self.connection_name = connection_name
self.project_name = project_name
+ self.implicit = implicit
def __repr__(self):
return '<ZuulRole %s %s>' % (self.project_name, self.target_name)
@@ -715,6 +717,8 @@
def __eq__(self, other):
if not isinstance(other, ZuulRole):
return False
+ # Implicit is not consulted for equality so that we can handle
+ # implicit to explicit conversions.
return (super(ZuulRole, self).__eq__(other) and
self.connection_name == other.connection_name and
self.project_name == other.project_name)
@@ -725,6 +729,7 @@
d['type'] = 'zuul'
d['connection'] = self.connection_name
d['project'] = self.project_name
+ d['implicit'] = self.implicit
return d
@@ -867,11 +872,31 @@
self.run = self.implied_run
def addRoles(self, roles):
- newroles = list(self.roles)
+ newroles = []
+ # Start with a copy of the existing roles, but if any of them
+ # are implicit roles which are identified as explicit in the
+ # new roles list, replace them with the explicit version.
+ changed = False
+ for existing_role in self.roles:
+ if existing_role in roles:
+ new_role = roles[roles.index(existing_role)]
+ else:
+ new_role = None
+ if (new_role and
+ isinstance(new_role, ZuulRole) and
+ isinstance(existing_role, ZuulRole) and
+ existing_role.implicit and not new_role.implicit):
+ newroles.append(new_role)
+ changed = True
+ else:
+ newroles.append(existing_role)
+ # Now add the new roles.
for role in reversed(roles):
if role not in newroles:
newroles.insert(0, role)
- self.roles = tuple(newroles)
+ changed = True
+ if changed:
+ self.roles = tuple(newroles)
def updateVariables(self, other_vars):
v = self.variables
@@ -1673,7 +1698,7 @@
url = default_url or build.url or job.name
return (result, url)
- def formatJSON(self):
+ def formatJSON(self, websocket_url=None):
ret = {}
ret['active'] = self.active
ret['live'] = self.live
@@ -1710,11 +1735,20 @@
remaining = None
result = None
build_url = None
+ finger_url = None
report_url = None
worker = None
if build:
result = build.result
- build_url = build.url
+ finger_url = build.url
+ # TODO(tobiash): add support for custom web root
+ urlformat = 'static/stream.html?' \
+ 'uuid={build.uuid}&' \
+ 'logfile=console.log'
+ if websocket_url:
+ urlformat += '&websocket_url={websocket_url}'
+ build_url = urlformat.format(
+ build=build, websocket_url=websocket_url)
(unused, report_url) = self.formatJobResult(job)
if build.start_time:
if build.end_time:
@@ -1740,6 +1774,7 @@
'elapsed_time': elapsed,
'remaining_time': remaining,
'url': build_url,
+ 'finger_url': finger_url,
'report_url': report_url,
'result': result,
'voting': job.voting,
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index e5e7f87..2217b0b 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -889,6 +889,7 @@
data = {}
data['zuul_version'] = self.zuul_version
+ websocket_url = get_default(self.config, 'web', 'websocket_url', None)
if self._pause:
ret = '<p><b>Queue only mode:</b> preparing to '
@@ -912,5 +913,5 @@
data['pipelines'] = pipelines
tenant = self.abide.tenants.get(tenant_name)
for pipeline in tenant.layout.pipelines.values():
- pipelines.append(pipeline.formatStatusJSON())
+ pipelines.append(pipeline.formatStatusJSON(websocket_url))
return json.dumps(data)
diff --git a/zuul/web.py b/zuul/web/__init__.py
similarity index 93%
rename from zuul/web.py
rename to zuul/web/__init__.py
index ab16e11..faf22b5 100644
--- a/zuul/web.py
+++ b/zuul/web/__init__.py
@@ -18,6 +18,7 @@
import asyncio
import json
import logging
+import os
import uvloop
import aiohttp
@@ -25,6 +26,8 @@
import zuul.rpcclient
+STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static')
+
class LogStreamingHandler(object):
log = logging.getLogger("zuul.web.LogStreamingHandler")
@@ -39,11 +42,11 @@
self.ssl_ca = ssl_ca
def _getPortLocation(self, job_uuid):
- '''
+ """
Query Gearman for the executor running the given job.
:param str job_uuid: The job UUID we want to stream.
- '''
+ """
# TODO: Fetch the entire list of uuid/file/server/ports once and
# share that, and fetch a new list on cache misses perhaps?
# TODO: Avoid recreating a client for each request.
@@ -55,14 +58,14 @@
return ret
async def _fingerClient(self, ws, server, port, job_uuid):
- '''
+ """
Create a client to connect to the finger streamer and pull results.
:param aiohttp.web.WebSocketResponse ws: The websocket response object.
:param str server: The executor server running the job.
:param str port: The executor server port.
:param str job_uuid: The job UUID to stream.
- '''
+ """
self.log.debug("Connecting to finger server %s:%s", server, port)
reader, writer = await asyncio.open_connection(host=server, port=port,
loop=self.event_loop)
@@ -82,12 +85,12 @@
return
async def _streamLog(self, ws, request):
- '''
+ """
Stream the log for the requested job back to the client.
:param aiohttp.web.WebSocketResponse ws: The websocket response object.
:param dict request: The client request parameters.
- '''
+ """
for key in ('uuid', 'logfile'):
if key not in request:
return (4000, "'{key}' missing from request payload".format(
@@ -112,11 +115,11 @@
return (1000, "No more data")
async def processRequest(self, request):
- '''
+ """
Handle a client websocket request for log streaming.
:param aiohttp.web.Request request: The client request.
- '''
+ """
try:
ws = web.WebSocketResponse()
await ws.prepare(request)
@@ -161,6 +164,8 @@
self.ssl_key = ssl_key
self.ssl_cert = ssl_cert
self.ssl_ca = ssl_ca
+ self.event_loop = None
+ self.term = None
async def _handleWebsocket(self, request):
handler = LogStreamingHandler(self.event_loop,
@@ -169,7 +174,7 @@
return await handler.processRequest(request)
def run(self, loop=None):
- '''
+ """
Run the websocket daemon.
Because this method can be the target of a new thread, we need to
@@ -178,9 +183,9 @@
:param loop: The event loop to use. If not supplied, the default main
thread event loop is used. This should be supplied if ZuulWeb
is run within a separate (non-main) thread.
- '''
+ """
routes = [
- ('GET', '/console-stream', self._handleWebsocket)
+ ('GET', '/console-stream', self._handleWebsocket),
]
self.log.debug("ZuulWeb starting")
@@ -195,6 +200,7 @@
app = web.Application()
for method, path, handler in routes:
app.router.add_route(method, path, handler)
+ app.router.add_static('/static', STATIC_DIR)
handler = app.make_handler(loop=self.event_loop)
# create the server
@@ -224,7 +230,8 @@
loop.close()
def stop(self):
- self.event_loop.call_soon_threadsafe(self.term.set_result, True)
+ if self.event_loop and self.term:
+ self.event_loop.call_soon_threadsafe(self.term.set_result, True)
if __name__ == "__main__":
diff --git a/zuul/web/static/stream.html b/zuul/web/static/stream.html
new file mode 100644
index 0000000..dbeb66b
--- /dev/null
+++ b/zuul/web/static/stream.html
@@ -0,0 +1,114 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
+ "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+ <head>
+ <style type="text/css">
+
+ body {
+ font-family: monospace;
+ background-color: black;
+ color: lightgrey;
+ }
+
+ #overlay {
+ position: fixed;
+ top: 5px;
+ right: 5px;
+ background-color: darkgrey;
+ color: black;
+ }
+
+ pre {
+ white-space: pre;
+ margin: 0px 10px;
+ }
+ </style>
+
+ <script type="text/javascript">
+
+ function escapeLog(text) {
+ var pattern = /[<>&"']/g;
+
+ return text.replace(pattern, function(match) {
+ return '&#' + match.charCodeAt(0) + ';';
+ });
+ }
+
+ window.onload = function() {
+
+ pageUpdateInMS = 250;
+ var receiveBuffer = "";
+ var websocket_url = null
+
+ setInterval(function() {
+ console.log("autoScroll");
+ if (receiveBuffer != "") {
+ document.getElementById('pagecontent').innerHTML += receiveBuffer;
+ receiveBuffer = "";
+ if (document.getElementById('autoscroll').checked) {
+ window.scrollTo(0, document.body.scrollHeight);
+ }
+ }
+ }, pageUpdateInMS);
+
+ var url = new URL(window.location);
+
+ var params = {
+ uuid: url.searchParams.get('uuid')
+ }
+ document.getElementById('pagetitle').innerHTML = params['uuid'];
+ if (url.searchParams.has('logfile')) {
+ params['logfile'] = url.searchParams.get('logfile');
+ var logfile_suffix = "(" + params['logfile'] + ")";
+ document.getElementById('pagetitle').innerHTML += logfile_suffix;
+ }
+ if (url.searchParams.has('websocket_url')) {
+ params['websocket_url'] = url.searchParams.get('websocket_url');
+ } else {
+ // Websocket doesn't accept relative urls so construct an
+ // absolute one.
+ var protocol = '';
+ if (url['protocol'] == 'https:') {
+ protocol = 'wss://';
+ } else {
+ protocol = 'ws://';
+ }
+ path = url['pathname'].replace(/static\/.*$/g, '') + 'console-stream';
+ params['websocket_url'] = protocol + url['host'] + path;
+ }
+ var ws = new WebSocket(params['websocket_url']);
+
+ ws.onmessage = function(event) {
+ console.log("onmessage");
+ receiveBuffer = receiveBuffer + escapeLog(event.data);
+ };
+
+ ws.onopen = function(event) {
+ console.log("onopen");
+ ws.send(JSON.stringify(params));
+ };
+
+ ws.onclose = function(event) {
+ console.log("onclose");
+ receiveBuffer = receiveBuffer + "\n--- END OF STREAM ---\n";
+ };
+
+ };
+
+ </script>
+
+ <title id="pagetitle"></title>
+ </head>
+
+ <body>
+
+ <div id="overlay">
+ <form>
+ <input type="checkbox" id="autoscroll" checked> autoscroll
+ </form>
+ </div>
+
+ <pre id="pagecontent"></pre>
+
+ </body>
+</html>