Merge "Validate top level of the layout configuration, too"
diff --git a/README.rst b/README.rst
index 1b227e7..ff4d938 100644
--- a/README.rst
+++ b/README.rst
@@ -9,7 +9,7 @@
To browse the latest code, see: https://git.openstack.org/cgit/openstack-infra/zuul/tree/
To clone the latest code, use `git clone git://git.openstack.org/openstack-infra/zuul`
-Bugs are handled at: https://launchpad.net/zuul
+Bugs are handled at: https://storyboard.openstack.org/#!/project/679
Code reviews are, as you might expect, handled by gerrit. The gerrit they
use is http://review.openstack.org
diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst
index dd650f2..5b745e6 100644
--- a/doc/source/triggers.rst
+++ b/doc/source/triggers.rst
@@ -34,6 +34,10 @@
be added to Gerrit. Zuul is very flexible and can take advantage of
those.
+If using Gerrit 2.7 or later, make sure the user is a member of a group
+that is granted the ``Stream Events`` permission, otherwise it will not
+be able to invoke the ``gerrit stream-events`` command over SSH.
+
Timer
-----
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index 6cb5d59..6e9fb05 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -86,6 +86,8 @@
zuul
""""
+.. _layout_config:
+
**layout_config**
Path to layout config file. Used by zuul-server only.
``layout_config=/etc/zuul/layout.yaml``
@@ -272,10 +274,12 @@
- python-file: local_functions.py
**python-file**
- The path to a python file. The file will be loaded and objects that
- it defines will be placed in a special environment which can be
- referenced in the Zuul configuration. Currently only the
- parameter-function attribute of a Job uses this feature.
+ The path to a python file (either an absolute path or relative to the
+ directory name of :ref:`layout_config <layout_config>`). The
+ file will be loaded and objects that it defines will be placed in a
+ special environment which can be referenced in the Zuul configuration.
+ Currently only the parameter-function attribute of a Job uses this
+ feature.
Pipelines
"""""""""
@@ -462,7 +466,8 @@
This may be used for any event. It requires that a certain kind
of approval be present for the current patchset of the change (the
approval could be added by the event in question). It follows the
- same syntax as the "approval" pipeline requirement below.
+ same syntax as the :ref:`"approval" pipeline requirement below
+ <pipeline-require-approval>`.
**timer**
This trigger will run based on a cron-style time specification.
@@ -497,7 +502,8 @@
This may be used for any event. It requires that a certain kind
of approval be present for the current patchset of the change (the
approval could be added by the event in question). It follows the
- same syntax as the "approval" pipeline requirement below.
+ same syntax as the :ref:`"approval" pipeline requirement below
+ <pipeline-require-approval>`.
**require**
@@ -507,6 +513,8 @@
the conditions specified here must be met or the item will not be
enqueued.
+.. _pipeline-require-approval:
+
**approval**
This requires that a certain kind of approval be present for the
current patchset of the change (the approval could be added by the
@@ -1045,3 +1053,10 @@
If you send a SIGUSR2 to the zuul-server process, Zuul will dump a stack
trace for each running thread into its debug log. This is useful for
tracking down deadlock or otherwise slow threads.
+
+When `yappi <https://code.google.com/p/yappi/>`_ (Yet Another Python
+Profiler) is available, additional functions' and threads' stats are
+emitted as well. The first SIGUSR2 will enable yappi, on the second
+SIGUSR2 it dumps the information collected, resets all yappi state and
+stops profiling. This is to minimize the impact of yappi on a running
+system.
diff --git a/requirements.txt b/requirements.txt
index 50726c0..4c85447 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,7 @@
Paste
WebOb>=1.2.3,<1.3
paramiko>=1.8.0
-GitPython>=0.3.2.RC1
+GitPython>=0.3.2.1
lockfile>=0.8
ordereddict
python-daemon
diff --git a/test-requirements.txt b/test-requirements.txt
index 5192de7..c68b2db 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,9 +1,8 @@
hacking>=0.9.2,<0.10
coverage>=3.6
-sphinx>=1.1.2,<1.2
+sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
sphinxcontrib-blockdiag>=0.5.5
-docutils==0.9.1
discover
fixtures>=0.3.14
python-keystoneclient>=0.4.2
diff --git a/tests/base.py b/tests/base.py
index 46c7087..1cafc46 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -40,6 +40,7 @@
import six.moves.urllib.parse as urlparse
import statsd
import testtools
+from git import GitCommandError
import zuul.scheduler
import zuul.webapp
@@ -1031,9 +1032,12 @@
def ref_has_change(self, ref, change):
path = os.path.join(self.git_root, change.project)
repo = git.Repo(path)
- for commit in repo.iter_commits(ref):
- if commit.message.strip() == ('%s-1' % change.subject):
- return True
+ try:
+ for commit in repo.iter_commits(ref):
+ if commit.message.strip() == ('%s-1' % change.subject):
+ return True
+ except GitCommandError:
+ pass
return False
def job_has_changes(self, *args):
diff --git a/tests/fixtures/layouts/bad_misplaced_ref.yaml b/tests/fixtures/layouts/bad_misplaced_ref.yaml
new file mode 100644
index 0000000..f009c39
--- /dev/null
+++ b/tests/fixtures/layouts/bad_misplaced_ref.yaml
@@ -0,0 +1,13 @@
+pipelines:
+ - name: 'check'
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ ref: /some/ref/path
+
+projects:
+ - name: org/project
+ merge-mode: cherry-pick
+ check:
+ - project-check
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index a895f24..8922161 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -97,7 +97,7 @@
zuul_missing = [zuul_opt for zuul_opt, val in vars(args).items()
if zuul_opt.startswith('zuul') and val is None]
if zuul_missing:
- parser.error(("Some Zuul parameters are not properly set:\n\t%s\n"
+ parser.error(("Some Zuul parameters are not set:\n\t%s\n"
"Define them either via environment variables or "
"using options above." %
"\n\t".join(sorted(zuul_missing))))
diff --git a/zuul/launcher/gearman.py b/zuul/launcher/gearman.py
index 57db7dc..564a554 100644
--- a/zuul/launcher/gearman.py
+++ b/zuul/launcher/gearman.py
@@ -128,7 +128,7 @@
for connection in self.active_connections:
try:
req = gear.StatusAdminRequest()
- connection.sendAdminRequest(req)
+ connection.sendAdminRequest(req, timeout=300)
except Exception:
self.log.exception("Exception while checking functions")
continue
@@ -203,7 +203,7 @@
for connection in self.gearman.active_connections:
try:
req = gear.StatusAdminRequest()
- connection.sendAdminRequest(req)
+ connection.sendAdminRequest(req, timeout=300)
except Exception:
self.log.exception("Exception while checking functions")
continue
@@ -361,15 +361,16 @@
precedence = gear.PRECEDENCE_LOW
try:
- self.gearman.submitJob(gearman_job, precedence=precedence)
+ self.gearman.submitJob(gearman_job, precedence=precedence,
+ timeout=300)
except Exception:
self.log.exception("Unable to submit job to Gearman")
self.onBuildCompleted(gearman_job, 'EXCEPTION')
return build
if not gearman_job.handle:
- self.log.error("No job handle was received for %s after 30 seconds"
- " marking as lost." %
+ self.log.error("No job handle was received for %s after"
+ " 300 seconds; marking as lost." %
gearman_job)
self.onBuildCompleted(gearman_job, 'NO_HANDLE')
@@ -467,7 +468,7 @@
job = build.__gearman_job
req = gear.CancelJobAdminRequest(job.handle)
- job.connection.sendAdminRequest(req)
+ job.connection.sendAdminRequest(req, timeout=300)
self.log.debug("Response to cancel build %s request: %s" %
(build, req.response.strip()))
if req.response.startswith("OK"):
@@ -486,7 +487,8 @@
json.dumps(data), unique=stop_uuid)
self.meta_jobs[stop_uuid] = stop_job
self.log.debug("Submitting stop job: %s", stop_job)
- self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH)
+ self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
+ timeout=300)
return True
def setBuildDescription(self, build, desc):
@@ -507,7 +509,8 @@
desc_job = gear.Job(name, json.dumps(data), unique=desc_uuid)
self.meta_jobs[desc_uuid] = desc_job
self.log.debug("Submitting describe job: %s", desc_job)
- self.gearman.submitJob(desc_job, precedence=gear.PRECEDENCE_LOW)
+ self.gearman.submitJob(desc_job, precedence=gear.PRECEDENCE_LOW,
+ timeout=300)
return True
def lookForLostBuilds(self):
diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py
index 4f3ff00..5a32c76 100644
--- a/zuul/layoutvalidator.py
+++ b/zuul/layoutvalidator.py
@@ -18,6 +18,7 @@
import voluptuous as v
import string
+from zuul.trigger import gerrit
# Several forms accept either a single item or a list, this makes
# specifying that in the schema easy (and explicit).
@@ -274,3 +275,7 @@
if 'project-templates' in data:
self.checkDuplicateNames(
data['project-templates'], ['project-templates'])
+ for pipeline in data['pipelines']:
+ if 'gerrit' in pipeline['trigger']:
+ gerrit.validate_trigger(pipeline['trigger'])
+
diff --git a/zuul/lib/gerrit.py b/zuul/lib/gerrit.py
index 52e6057..5aad953 100644
--- a/zuul/lib/gerrit.py
+++ b/zuul/lib/gerrit.py
@@ -145,21 +145,34 @@
return data
def simpleQuery(self, query):
- args = '--current-patch-set'
- cmd = 'gerrit query --format json %s %s' % (
- args, query)
- out, err = self._ssh(cmd)
- if not out:
- return False
- lines = out.split('\n')
- if not lines:
- return False
- data = [json.loads(line) for line in lines[:-1]]
- if not data:
- return False
- self.log.debug("Received data from Gerrit query: \n%s" %
- (pprint.pformat(data)))
- return data
+ def _query_chunk(query):
+ args = '--current-patch-set'
+
+ cmd = 'gerrit query --format json %s %s' % (
+ args, query)
+ out, err = self._ssh(cmd)
+ if not out:
+ return False
+ lines = out.split('\n')
+ if not lines:
+ return False
+ data = [json.loads(line) for line in lines
+ if "sortKey" in line]
+ if not data:
+ return False
+ self.log.debug("Received data from Gerrit query: \n%s" %
+ (pprint.pformat(data)))
+ return data
+
+ # gerrit returns 500 results by default, so implement paging
+ # for large projects like nova
+ alldata = []
+ chunk = _query_chunk(query)
+ while(chunk):
+ alldata.extend(chunk)
+ sortkey = "resume_sortkey:'%s'" % chunk[-1]["sortKey"]
+ chunk = _query_chunk("%s %s" % (query, sortkey))
+ return alldata
def _open(self):
client = paramiko.SSHClient()
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 8c41563..8d8f7ee 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -89,7 +89,8 @@
json.dumps(data),
unique=uuid)
self.build_sets[uuid] = build_set
- self.gearman.submitJob(job, precedence=precedence)
+ self.gearman.submitJob(job, precedence=precedence,
+ timeout=300)
def mergeChanges(self, items, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
diff --git a/zuul/model.py b/zuul/model.py
index b03bbb0..67ce8be 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -709,6 +709,10 @@
ret['project'] = changeish.project.name
ret['enqueue_time'] = int(self.enqueue_time * 1000)
ret['jobs'] = []
+ if hasattr(changeish, 'owner'):
+ ret['owner'] = changeish.owner
+ else:
+ ret['owner'] = None
max_remaining = 0
for job in self.pipeline.getJobs(changeish):
now = time.time()
@@ -857,6 +861,7 @@
self.approvals = []
self.open = None
self.status = None
+ self.owner = None
def _id(self):
return '%s,%s' % (self.number, self.patchset)
diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py
index 7f572be..f43c3b9 100644
--- a/zuul/rpcclient.py
+++ b/zuul/rpcclient.py
@@ -38,7 +38,7 @@
job = gear.Job(name,
json.dumps(data),
unique=str(time.time()))
- self.gearman.submitJob(job)
+ self.gearman.submitJob(job, timeout=300)
self.log.debug("Waiting for job completion")
while not job.complete:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 9effcb8..25f3ae0 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -226,7 +226,7 @@
if 'python-file' in include:
fn = include['python-file']
if not os.path.isabs(fn):
- base = os.path.dirname(config_path)
+ base = os.path.dirname(os.path.realpath(config_path))
fn = os.path.join(base, fn)
fn = os.path.expanduser(fn)
execfile(fn, config_env)
@@ -810,7 +810,7 @@
try:
project = self.layout.projects.get(event.project_name)
if not project:
- self.log.warning("Project %s not found" % event.project_name)
+ self.log.debug("Project %s not found" % event.project_name)
return
for pipeline in self.layout.pipelines.values():
diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py
index 6966488..0c0a376 100644
--- a/zuul/trigger/gerrit.py
+++ b/zuul/trigger/gerrit.py
@@ -16,6 +16,7 @@
import threading
import time
import urllib2
+import voluptuous
from zuul.lib import gerrit
from zuul.model import TriggerEvent, Change, Ref, NullChange
@@ -330,7 +331,7 @@
self.log.debug("Running query %s to get project open changes" % (query,))
data = self.gerrit.simpleQuery(query)
changes = []
- for record in data[:-1]:
+ for record in data:
try:
changes.append(self._getChange(record['number'],
record['currentPatchSet']['number']))
@@ -371,6 +372,7 @@
change.approvals = data['currentPatchSet'].get('approvals', [])
change.open = data['open']
change.status = data['status']
+ change.owner = data['owner']
if change.is_merged:
# This change is merged, so we don't need to look any further
@@ -411,3 +413,14 @@
if sha:
url += ';a=commitdiff;h=' + sha
return url
+
+
+def validate_trigger(trigger_data):
+ """Validates the layout's trigger data."""
+ events_with_ref = ('ref-updated', )
+ for event in trigger_data['gerrit']:
+ if event['event'] not in events_with_ref and event.get('ref', False):
+ raise voluptuous.Invalid(
+ "The event %s does not include ref information, Zuul cannot "
+ "use ref filter 'ref: %s'" % (event['event'], event['ref']))
+