Merge "Log execution phase and include information" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 26a85b2..a0de922 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -32,24 +32,25 @@
Client connection information for gearman.
-**server**
- Hostname or IP address of the Gearman server.
- ``server=gearman.example.com`` (required)
+**server** (required)
+ Hostname or IP address of the Gearman server::
+
+ server=gearman.example.com
**port**
- Port on which the Gearman server is listening.
- ``port=4730`` (optional)
+ Port on which the Gearman server is listening::
+
+ port=4730
**ssl_ca**
- Optional: An openssl file containing a set of concatenated
- “certification authority” certificates in PEM formet.
+ An openssl file containing a set of concatenated “certification
+ authority” certificates in PEM formet.
**ssl_cert**
- Optional: An openssl file containing the client public certificate in
- PEM format.
+ An openssl file containing the client public certificate in PEM format.
**ssl_key**
- Optional: An openssl file containing the client private key in PEM format.
+ An openssl file containing the client private key in PEM format.
zookeeper
"""""""""
@@ -60,7 +61,9 @@
**hosts**
A list of zookeeper hosts for Zuul to use when communicating with
- Nodepool. ``hosts=zk1.example.com,zk2.example.com,zk3.example.com``
+ Nodepool::
+
+ hosts=zk1.example.com,zk2.example.com,zk3.example.com
Scheduler
@@ -85,66 +88,76 @@
than connecting to an external one.
**start**
- Whether to start the internal Gearman server (default: False).
- ``start=true``
+ Whether to start the internal Gearman server (default: False)::
+
+ start=true
**listen_address**
- IP address or domain name on which to listen (default: all addresses).
- ``listen_address=127.0.0.1``
+ IP address or domain name on which to listen (default: all addresses)::
+
+ listen_address=127.0.0.1
**log_config**
- Path to log config file for internal Gearman server.
- ``log_config=/etc/zuul/gearman-logging.yaml``
+ Path to log config file for internal Gearman server::
+
+ log_config=/etc/zuul/gearman-logging.yaml
**ssl_ca**
- Optional: An openssl file containing a set of concatenated “certification authority” certificates
- in PEM formet.
+ An openssl file containing a set of concatenated “certification authority”
+ certificates in PEM formet.
**ssl_cert**
- Optional: An openssl file containing the server public certificate in PEM format.
+ An openssl file containing the server public certificate in PEM format.
**ssl_key**
- Optional: An openssl file containing the server private key in PEM format.
+ An openssl file containing the server private key in PEM format.
webapp
""""""
**listen_address**
- IP address or domain name on which to listen (default: 0.0.0.0).
- ``listen_address=127.0.0.1``
+ IP address or domain name on which to listen (default: 0.0.0.0)::
+
+ listen_address=127.0.0.1
**port**
- Port on which the webapp is listening (default: 8001).
- ``port=8008``
+ Port on which the webapp is listening (default: 8001)::
+
+ port=8008
**status_expiry**
- Zuul will cache the status.json file for this many seconds. This is an
- optional value and ``1`` is used by default.
- ``status_expiry=1``
+ Zuul will cache the status.json file for this many seconds (default: 1)::
+
+ status_expiry=1
**status_url**
URL that will be posted in Zuul comments made to changes when
- starting jobs for a change. Used by zuul-scheduler only.
- ``status_url=https://zuul.example.com/status``
+ starting jobs for a change. Used by zuul-scheduler only::
+
+ status_url=https://zuul.example.com/status
scheduler
"""""""""
**tenant_config**
- Path to tenant config file.
- ``layout_config=/etc/zuul/tenant.yaml``
+ Path to tenant config file::
+
+ layout_config=/etc/zuul/tenant.yaml
**log_config**
- Path to log config file.
- ``log_config=/etc/zuul/scheduler-logging.yaml``
+ Path to log config file::
+
+ log_config=/etc/zuul/scheduler-logging.yaml
**pidfile**
- Path to PID lock file.
- ``pidfile=/var/run/zuul/scheduler.pid``
+ Path to PID lock file::
+
+ pidfile=/var/run/zuul/scheduler.pid
**state_dir**
- Path to directory that Zuul should save state to.
- ``state_dir=/var/lib/zuul``
+ Path to directory that Zuul should save state to::
+
+ state_dir=/var/lib/zuul
Operation
~~~~~~~~~
@@ -186,24 +199,29 @@
""""""
**git_dir**
- Directory that Zuul should clone local git repositories to.
- ``git_dir=/var/lib/zuul/git``
+ Directory that Zuul should clone local git repositories to::
+
+ git_dir=/var/lib/zuul/git
**git_user_email**
- Optional: Value to pass to `git config user.email`.
- ``git_user_email=zuul@example.com``
+ Value to pass to `git config user.email`::
+
+ git_user_email=zuul@example.com
**git_user_name**
- Optional: Value to pass to `git config user.name`.
- ``git_user_name=zuul``
+ Value to pass to `git config user.name`::
+
+ git_user_name=zuul
**log_config**
- Path to log config file for the merger process.
- ``log_config=/etc/zuul/logging.yaml``
+ Path to log config file for the merger process::
+
+ log_config=/etc/zuul/logging.yaml
**pidfile**
- Path to PID lock file for the merger process.
- ``pidfile=/var/run/zuul-merger/merger.pid``
+ Path to PID lock file for the merger process::
+
+ pidfile=/var/run/zuul-merger/merger.pid
Operation
~~~~~~~~~
@@ -273,37 +291,44 @@
""""""""
**finger_port**
- Port to use for finger log streamer.
- ``finger_port=79``
+ Port to use for finger log streamer::
+
+ finger_port=79
**git_dir**
- Directory that Zuul should clone local git repositories to.
- ``git_dir=/var/lib/zuul/git``
+ Directory that Zuul should clone local git repositories to::
+
+ git_dir=/var/lib/zuul/git
**log_config**
- Path to log config file for the executor process.
- ``log_config=/etc/zuul/logging.yaml``
+ Path to log config file for the executor process::
+
+ log_config=/etc/zuul/logging.yaml
**private_key_file**
- SSH private key file to be used when logging into worker nodes.
- ``private_key_file=~/.ssh/id_rsa``
+ SSH private key file to be used when logging into worker nodes::
+
+ private_key_file=~/.ssh/id_rsa
**user**
User ID for the zuul-executor process. In normal operation as a daemon,
the executor should be started as the ``root`` user, but it will drop
- privileges to this user during startup.
- ``user=zuul``
+ privileges to this user during startup::
+
+ user=zuul
merger
""""""
**git_user_email**
- Optional: Value to pass to `git config user.email`.
- ``git_user_email=zuul@example.com``
+ Value to pass to `git config user.email`::
+
+ git_user_email=zuul@example.com
**git_user_name**
- Optional: Value to pass to `git config user.name`.
- ``git_user_name=zuul``
+ Value to pass to `git config user.name`::
+
+ git_user_name=zuul
Operation
~~~~~~~~~
@@ -341,20 +366,24 @@
"""
**listen_address**
- IP address or domain name on which to listen (default: 127.0.0.1).
- ``listen_address=127.0.0.1``
+ IP address or domain name on which to listen (default: 127.0.0.1)::
+
+ listen_address=127.0.0.1
**log_config**
- Path to log config file for the web server process.
- ``log_config=/etc/zuul/logging.yaml``
+ Path to log config file for the web server process::
+
+ log_config=/etc/zuul/logging.yaml
**pidfile**
- Path to PID lock file for the web server process.
- ``pidfile=/var/run/zuul-web/zuul-web.pid``
+ Path to PID lock file for the web server process::
+
+ pidfile=/var/run/zuul-web/zuul-web.pid
**port**
- Port to use for web server process.
- ``port=9000``
+ Port to use for web server process::
+
+ port=9000
Operation
~~~~~~~~~
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 0b2b5d4..5a5b7bd 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -662,32 +662,35 @@
of attempts to make before an error is reported. Default: 3.
**pre-run**
- The name of a playbook or list of playbooks to run before the main
- body of a job. The playbook is expected to reside in the
- `playbooks/` directory of the project where the job is defined.
+ The name of a playbook or list of playbooks without file extension
+ to run before the main body of a job. The full path to the playbook
+ in the repo where the job is defined is expected.
When a job inherits from a parent, the child's pre-run playbooks are
run after the parent's. See :ref:`job` for more information.
**post-run**
- The name of a playbook or list of playbooks to run after the main
- body of a job. The playbook is expected to reside in the
- `playbooks/` directory of the project where the job is defined.
+ The name of a playbook or list of playbooks without file extension
+ to run after the main body of a job. The full path to the playbook
+ in the repo where the job is defined is expected.
When a job inherits from a parent, the child's post-run playbooks
are run before the parent's. See :ref:`job` for more information.
**run**
- The name of the main playbook for this job. This parameter is not
- normally necessary, as it defaults to the name of the job. However,
- if a playbook with a different name is needed, it can be specified
- here. The playbook is expected to reside in the `playbooks/`
- directory of the project where the job is defined. When a child
- inherits from a parent, a playbook with the name of the child job is
- implicitly searched first, before falling back on the playbook used
- by the parent job (unless the child job specifies a ``run``
- attribute, in which case that value is used). Default: the name of
- the job.
+ The name of the main playbook for this job. This parameter is
+ not normally necessary, as it defaults to a playbook with the
+ same name as the job inside of the `playbooks/` directory (e.g.,
+ the `foo` job would default to `playbooks/foo`. However, if a
+ playbook with a different name is needed, it can be specified
+ here. The file extension is not required, but the full path
+ within the repo is. When a child inherits from a parent, a
+ playbook with the name of the child job is implicitly searched
+ first, before falling back on the playbook used by the parent
+ job (unless the child job specifies a ``run`` attribute, in which
+ case that value is used). Example::
+
+ run: playbooks/<name of the job>
**roles**
A list of Ansible roles to prepare for the job. Because a job runs
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 5637552..58f3371 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -101,3 +101,28 @@
.. TODO: describe standard lib and link to published docs for it.
+Return Values
+-------------
+
+The job may return some values to Zuul to affect its behavior. To
+return a value, use the *zuul_return* Ansible module in a job
+playbook. For example::
+
+ tasks:
+ - zuul_return:
+ data:
+ foo: bar
+
+Will return the dictionary "{'foo': 'bar'}" to Zuul.
+
+.. TODO: xref to section describing formatting
+
+Several uses of these values are planned, but the only currently
+implemented use is to set the log URL for a build. To do so, set the
+**zuul.log_url** value. For example::
+
+ tasks:
+ - zuul_return:
+ data:
+ zuul:
+ log_url: http://logs.example.com/path/to/build/logs
diff --git a/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml b/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml
new file mode 100644
index 0000000..b92ff5c
--- /dev/null
+++ b/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml
@@ -0,0 +1,6 @@
+- hosts: localhost
+ tasks:
+ - zuul_return:
+ data:
+ zuul:
+ log_url: test/log/url
diff --git a/tests/fixtures/config/data-return/git/common-config/zuul.yaml b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
new file mode 100644
index 0000000..8aea931
--- /dev/null
+++ b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
@@ -0,0 +1,22 @@
+- pipeline:
+ name: check
+ manager: independent
+ allow-secrets: true
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: data-return
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - data-return
diff --git a/tests/fixtures/config/data-return/git/org_project/README b/tests/fixtures/config/data-return/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/data-return/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/data-return/main.yaml b/tests/fixtures/config/data-return/main.yaml
new file mode 100644
index 0000000..208e274
--- /dev/null
+++ b/tests/fixtures/config/data-return/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 2b865cf..5d49d11 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -699,4 +699,20 @@
self.assertHistory([
dict(name='test1', result='SUCCESS', changes='1,1'),
dict(name='test2', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
+class TestDataReturn(AnsibleZuulTestCase):
+ tenant_config_file = 'config/data-return/main.yaml'
+
+ def test_data_return(self):
+ # This exercises a proposed change to a role being checked out
+ # and used.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='data-return', result='SUCCESS', changes='1,1'),
])
+ self.assertIn('- data-return test/log/url',
+ A.messages[-1])
diff --git a/zuul/ansible/library/zuul_return.py b/zuul/ansible/library/zuul_return.py
new file mode 100644
index 0000000..9f3332b
--- /dev/null
+++ b/zuul/ansible/library/zuul_return.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python
+
+# Copyright (c) 2017 Red Hat
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import json
+import tempfile
+
+
+def set_value(path, new_data, new_file):
+ workdir = os.path.dirname(path)
+ data = None
+ if os.path.exists(path):
+ with open(path, 'r') as f:
+ data = f.read()
+ if data:
+ data = json.loads(data)
+ else:
+ data = {}
+
+ if new_file:
+ with open(new_file, 'r') as f:
+ data.update(json.load(f))
+ if new_data:
+ data.update(new_data)
+
+ (f, tmp_path) = tempfile.mkstemp(dir=workdir)
+ try:
+ f = os.fdopen(f, 'w')
+ json.dump(data, f)
+ f.close()
+ os.rename(tmp_path, path)
+ except Exception:
+ os.unlink(tmp_path)
+ raise
+
+
+def main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ path=dict(required=False, type='str'),
+ data=dict(required=False, type='dict'),
+ file=dict(required=False, type='str'),
+ )
+ )
+
+ p = module.params
+ path = p['path']
+ if not path:
+ path = os.path.join(os.environ['ZUUL_JOBDIR'], 'work',
+ 'results.json')
+ set_value(path, p['data'], p['file'])
+ module.exit_json(changed=True, e=os.environ)
+
+from ansible.module_utils.basic import * # noqa
+from ansible.module_utils.basic import AnsibleModule
+
+if __name__ == '__main__':
+ main()
diff --git a/zuul/driver/sql/alembic_reporter/versions/20126015a87d_add_indexes.py b/zuul/driver/sql/alembic_reporter/versions/20126015a87d_add_indexes.py
new file mode 100644
index 0000000..3ac680d
--- /dev/null
+++ b/zuul/driver/sql/alembic_reporter/versions/20126015a87d_add_indexes.py
@@ -0,0 +1,56 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""add indexes
+
+Revision ID: 20126015a87d
+Revises: 1dd914d4a482
+Create Date: 2017-07-07 07:17:27.992040
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '20126015a87d'
+down_revision = '1dd914d4a482'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+
+BUILDSET_TABLE = 'zuul_buildset'
+BUILD_TABLE = 'zuul_build'
+
+
+def upgrade():
+ # To allow a dashboard to show a per-project view, optionally filtered
+ # by pipeline.
+ op.create_index(
+ 'project_pipeline_idx', BUILDSET_TABLE, ['project', 'pipeline'])
+
+ # To allow a dashboard to show a per-project-change view
+ op.create_index(
+ 'project_change_idx', BUILDSET_TABLE, ['project', 'change'])
+
+ # To allow a dashboard to show a per-change view
+ op.create_index('change_idx', BUILDSET_TABLE, ['change'])
+
+ # To allow a dashboard to show a job lib view. buildset_id is included
+ # so that it's a covering index and can satisfy the join back to buildset
+ # without an additional lookup.
+ op.create_index(
+ 'job_name_buildset_id_idx', BUILD_TABLE, ['job_name', 'buildset_id'])
+
+
+def downgrade():
+ pass
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index d17e47e..c36d569 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -107,7 +107,6 @@
class ExecutorClient(object):
log = logging.getLogger("zuul.ExecutorClient")
- negative_function_cache_ttl = 5
def __init__(self, config, sched):
self.config = config
@@ -125,8 +124,6 @@
self.cleanup_thread = GearmanCleanup(self)
self.cleanup_thread.start()
- self.function_cache = set()
- self.function_cache_time = 0
def stop(self):
self.log.debug("Stopping")
@@ -298,7 +295,7 @@
build.parameters = params
if job.name == 'noop':
- self.sched.onBuildCompleted(build, 'SUCCESS')
+ self.sched.onBuildCompleted(build, 'SUCCESS', {})
return build
gearman_job = gear.TextJob('executor:execute', json.dumps(params),
@@ -386,9 +383,10 @@
result = 'RETRY_LIMIT'
else:
build.retry = True
+ result_data = data.get('data', {})
self.log.info("Build %s complete, result %s" %
(job, result))
- self.sched.onBuildCompleted(build, result)
+ self.sched.onBuildCompleted(build, result, result_data)
# The test suite expects the build to be removed from the
# internal dict after it's added to the report queue.
del self.builds[job.unique]
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 3530f39..cdd082e 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -187,6 +187,9 @@
os.makedirs(self.ansible_root)
ssh_dir = os.path.join(self.work_root, '.ssh')
os.mkdir(ssh_dir, 0o700)
+ self.result_data_file = os.path.join(self.work_root, 'results.json')
+ with open(self.result_data_file, 'w'):
+ pass
self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
self.playbooks = [] # The list of candidate playbooks
@@ -835,12 +838,22 @@
self.job.sendWorkStatus(0, 100)
result = self.runPlaybooks(args)
+ data = self.getResultData()
+ result_data = json.dumps(dict(result=result,
+ data=data))
+ self.log.debug("Sending result: %s" % (result_data,))
+ self.job.sendWorkComplete(result_data)
- if result is None:
- self.job.sendWorkFail()
- return
- result = dict(result=result)
- self.job.sendWorkComplete(json.dumps(result))
+ def getResultData(self):
+ data = {}
+ try:
+ with open(self.jobdir.result_data_file) as f:
+ file_data = f.read()
+ if file_data:
+ data = json.loads(file_data)
+ except Exception:
+ self.log.exception("Unable to load result data:")
+ return data
def doMergeChanges(self, merger, items, repo_state):
ret = merger.mergeChanges(items, repo_state=repo_state)
@@ -1185,7 +1198,8 @@
all_vars['zuul']['executor'] = dict(
hostname=self.executor_server.hostname,
src_root=self.jobdir.src_root,
- log_root=self.jobdir.log_root)
+ log_root=self.jobdir.log_root,
+ result_data_file=self.jobdir.result_data_file)
nodes = self.getHostList(args)
inventory = make_inventory_dict(nodes, args['groups'], all_vars)
@@ -1277,6 +1291,7 @@
env_copy.update(self.ssh_agent.env)
env_copy['LOGNAME'] = 'zuul'
env_copy['ZUUL_JOB_OUTPUT_FILE'] = self.jobdir.job_output_file
+ env_copy['ZUUL_JOBDIR'] = self.jobdir.root
pythonpath = env_copy.get('PYTHONPATH')
if pythonpath:
pythonpath = [pythonpath]
diff --git a/zuul/model.py b/zuul/model.py
index 9d39a0c..ffbb70c 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1077,6 +1077,7 @@
self.uuid = uuid
self.url = None
self.result = None
+ self.result_data = {}
self.build_set = None
self.execute_time = time.time()
self.start_time = None
@@ -1095,7 +1096,9 @@
(self.uuid, self.job.name, self.worker))
def getSafeAttributes(self):
- return Attributes(uuid=self.uuid)
+ return Attributes(uuid=self.uuid,
+ result=self.result,
+ result_data=self.result_data)
class Worker(object):
@@ -1627,6 +1630,8 @@
if pattern:
url = self.formatUrlPattern(pattern, job, build)
if not url:
+ url = build.result_data.get('zuul', {}).get('log_url')
+ if not url:
url = build.url or job.name
return (result, url)
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index dd0846d..e5e7f87 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -273,10 +273,11 @@
self.wake_event.set()
self.log.debug("Done adding start event for build: %s" % build)
- def onBuildCompleted(self, build, result):
+ def onBuildCompleted(self, build, result, result_data):
self.log.debug("Adding complete event for build: %s result: %s" % (
build, result))
build.end_time = time.time()
+ build.result_data = result_data
# Note, as soon as the result is set, other threads may act
# upon this, even though the event hasn't been fully
# processed. Ensure that any other data from the event (eg,