Merge "Fix tests for GitPython>0.3.2.RC1"
diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst
index dd650f2..5b745e6 100644
--- a/doc/source/triggers.rst
+++ b/doc/source/triggers.rst
@@ -34,6 +34,10 @@
be added to Gerrit. Zuul is very flexible and can take advantage of
those.
+If using Gerrit 2.7 or later, make sure the user is a member of a group
+that is granted the ``Stream Events`` permission, otherwise it will not
+be able to invoke the ``gerrit stream-events`` command over SSH.
+
Timer
-----
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index 6cb5d59..6e9fb05 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -86,6 +86,8 @@
zuul
""""
+.. _layout_config:
+
**layout_config**
Path to layout config file. Used by zuul-server only.
``layout_config=/etc/zuul/layout.yaml``
@@ -272,10 +274,12 @@
- python-file: local_functions.py
**python-file**
- The path to a python file. The file will be loaded and objects that
- it defines will be placed in a special environment which can be
- referenced in the Zuul configuration. Currently only the
- parameter-function attribute of a Job uses this feature.
+ The path to a python file (either an absolute path or relative to the
+ directory name of :ref:`layout_config <layout_config>`). The
+ file will be loaded and objects that it defines will be placed in a
+ special environment which can be referenced in the Zuul configuration.
+ Currently only the parameter-function attribute of a Job uses this
+ feature.
Pipelines
"""""""""
@@ -462,7 +466,8 @@
This may be used for any event. It requires that a certain kind
of approval be present for the current patchset of the change (the
approval could be added by the event in question). It follows the
- same syntax as the "approval" pipeline requirement below.
+ same syntax as the :ref:`"approval" pipeline requirement below
+ <pipeline-require-approval>`.
**timer**
This trigger will run based on a cron-style time specification.
@@ -497,7 +502,8 @@
This may be used for any event. It requires that a certain kind
of approval be present for the current patchset of the change (the
approval could be added by the event in question). It follows the
- same syntax as the "approval" pipeline requirement below.
+ same syntax as the :ref:`"approval" pipeline requirement below
+ <pipeline-require-approval>`.
**require**
@@ -507,6 +513,8 @@
the conditions specified here must be met or the item will not be
enqueued.
+.. _pipeline-require-approval:
+
**approval**
This requires that a certain kind of approval be present for the
current patchset of the change (the approval could be added by the
@@ -1045,3 +1053,10 @@
If you send a SIGUSR2 to the zuul-server process, Zuul will dump a stack
trace for each running thread into its debug log. This is useful for
tracking down deadlock or otherwise slow threads.
+
+When `yappi <https://code.google.com/p/yappi/>`_ (Yet Another Python
+Profiler) is available, additional functions' and threads' stats are
+emitted as well. The first SIGUSR2 will enable yappi, on the second
+SIGUSR2 it dumps the information collected, resets all yappi state and
+stops profiling. This is to minimize the impact of yappi on a running
+system.
diff --git a/test-requirements.txt b/test-requirements.txt
index 5192de7..c68b2db 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,9 +1,8 @@
hacking>=0.9.2,<0.10
coverage>=3.6
-sphinx>=1.1.2,<1.2
+sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
sphinxcontrib-blockdiag>=0.5.5
-docutils==0.9.1
discover
fixtures>=0.3.14
python-keystoneclient>=0.4.2
diff --git a/tests/fixtures/layouts/bad_misplaced_ref.yaml b/tests/fixtures/layouts/bad_misplaced_ref.yaml
new file mode 100644
index 0000000..f009c39
--- /dev/null
+++ b/tests/fixtures/layouts/bad_misplaced_ref.yaml
@@ -0,0 +1,13 @@
+pipelines:
+ - name: 'check'
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ ref: /some/ref/path
+
+projects:
+ - name: org/project
+ merge-mode: cherry-pick
+ check:
+ - project-check
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index a895f24..8922161 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -97,7 +97,7 @@
zuul_missing = [zuul_opt for zuul_opt, val in vars(args).items()
if zuul_opt.startswith('zuul') and val is None]
if zuul_missing:
- parser.error(("Some Zuul parameters are not properly set:\n\t%s\n"
+ parser.error(("Some Zuul parameters are not set:\n\t%s\n"
"Define them either via environment variables or "
"using options above." %
"\n\t".join(sorted(zuul_missing))))
diff --git a/zuul/launcher/gearman.py b/zuul/launcher/gearman.py
index 57db7dc..564a554 100644
--- a/zuul/launcher/gearman.py
+++ b/zuul/launcher/gearman.py
@@ -128,7 +128,7 @@
for connection in self.active_connections:
try:
req = gear.StatusAdminRequest()
- connection.sendAdminRequest(req)
+ connection.sendAdminRequest(req, timeout=300)
except Exception:
self.log.exception("Exception while checking functions")
continue
@@ -203,7 +203,7 @@
for connection in self.gearman.active_connections:
try:
req = gear.StatusAdminRequest()
- connection.sendAdminRequest(req)
+ connection.sendAdminRequest(req, timeout=300)
except Exception:
self.log.exception("Exception while checking functions")
continue
@@ -361,15 +361,16 @@
precedence = gear.PRECEDENCE_LOW
try:
- self.gearman.submitJob(gearman_job, precedence=precedence)
+ self.gearman.submitJob(gearman_job, precedence=precedence,
+ timeout=300)
except Exception:
self.log.exception("Unable to submit job to Gearman")
self.onBuildCompleted(gearman_job, 'EXCEPTION')
return build
if not gearman_job.handle:
- self.log.error("No job handle was received for %s after 30 seconds"
- " marking as lost." %
+ self.log.error("No job handle was received for %s after"
+ " 300 seconds; marking as lost." %
gearman_job)
self.onBuildCompleted(gearman_job, 'NO_HANDLE')
@@ -467,7 +468,7 @@
job = build.__gearman_job
req = gear.CancelJobAdminRequest(job.handle)
- job.connection.sendAdminRequest(req)
+ job.connection.sendAdminRequest(req, timeout=300)
self.log.debug("Response to cancel build %s request: %s" %
(build, req.response.strip()))
if req.response.startswith("OK"):
@@ -486,7 +487,8 @@
json.dumps(data), unique=stop_uuid)
self.meta_jobs[stop_uuid] = stop_job
self.log.debug("Submitting stop job: %s", stop_job)
- self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH)
+ self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
+ timeout=300)
return True
def setBuildDescription(self, build, desc):
@@ -507,7 +509,8 @@
desc_job = gear.Job(name, json.dumps(data), unique=desc_uuid)
self.meta_jobs[desc_uuid] = desc_job
self.log.debug("Submitting describe job: %s", desc_job)
- self.gearman.submitJob(desc_job, precedence=gear.PRECEDENCE_LOW)
+ self.gearman.submitJob(desc_job, precedence=gear.PRECEDENCE_LOW,
+ timeout=300)
return True
def lookForLostBuilds(self):
diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py
index 9a4e00f..6969653 100644
--- a/zuul/layoutvalidator.py
+++ b/zuul/layoutvalidator.py
@@ -18,6 +18,7 @@
import voluptuous as v
import string
+from zuul.trigger import gerrit
# Several forms accept either a single item or a list, this makes
# specifying that in the schema easy (and explicit).
@@ -271,3 +272,7 @@
if 'project-templates' in data:
self.checkDuplicateNames(
data['project-templates'], ['project-templates'])
+ for pipeline in data['pipelines']:
+ if 'gerrit' in pipeline['trigger']:
+ gerrit.validate_trigger(pipeline['trigger'])
+
diff --git a/zuul/lib/gerrit.py b/zuul/lib/gerrit.py
index 52e6057..5aad953 100644
--- a/zuul/lib/gerrit.py
+++ b/zuul/lib/gerrit.py
@@ -145,21 +145,34 @@
return data
def simpleQuery(self, query):
- args = '--current-patch-set'
- cmd = 'gerrit query --format json %s %s' % (
- args, query)
- out, err = self._ssh(cmd)
- if not out:
- return False
- lines = out.split('\n')
- if not lines:
- return False
- data = [json.loads(line) for line in lines[:-1]]
- if not data:
- return False
- self.log.debug("Received data from Gerrit query: \n%s" %
- (pprint.pformat(data)))
- return data
+ def _query_chunk(query):
+ args = '--current-patch-set'
+
+ cmd = 'gerrit query --format json %s %s' % (
+ args, query)
+ out, err = self._ssh(cmd)
+ if not out:
+ return False
+ lines = out.split('\n')
+ if not lines:
+ return False
+ data = [json.loads(line) for line in lines
+ if "sortKey" in line]
+ if not data:
+ return False
+ self.log.debug("Received data from Gerrit query: \n%s" %
+ (pprint.pformat(data)))
+ return data
+
+ # gerrit returns 500 results by default, so implement paging
+ # for large projects like nova
+ alldata = []
+ chunk = _query_chunk(query)
+ while(chunk):
+ alldata.extend(chunk)
+ sortkey = "resume_sortkey:'%s'" % chunk[-1]["sortKey"]
+ chunk = _query_chunk("%s %s" % (query, sortkey))
+ return alldata
def _open(self):
client = paramiko.SSHClient()
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 8c41563..8d8f7ee 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -89,7 +89,8 @@
json.dumps(data),
unique=uuid)
self.build_sets[uuid] = build_set
- self.gearman.submitJob(job, precedence=precedence)
+ self.gearman.submitJob(job, precedence=precedence,
+ timeout=300)
def mergeChanges(self, items, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py
index 7f572be..f43c3b9 100644
--- a/zuul/rpcclient.py
+++ b/zuul/rpcclient.py
@@ -38,7 +38,7 @@
job = gear.Job(name,
json.dumps(data),
unique=str(time.time()))
- self.gearman.submitJob(job)
+ self.gearman.submitJob(job, timeout=300)
self.log.debug("Waiting for job completion")
while not job.complete:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 9effcb8..def7429 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -226,7 +226,7 @@
if 'python-file' in include:
fn = include['python-file']
if not os.path.isabs(fn):
- base = os.path.dirname(config_path)
+ base = os.path.dirname(os.path.realpath(config_path))
fn = os.path.join(base, fn)
fn = os.path.expanduser(fn)
execfile(fn, config_env)
diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py
index bf591c4..0c0a376 100644
--- a/zuul/trigger/gerrit.py
+++ b/zuul/trigger/gerrit.py
@@ -16,6 +16,7 @@
import threading
import time
import urllib2
+import voluptuous
from zuul.lib import gerrit
from zuul.model import TriggerEvent, Change, Ref, NullChange
@@ -330,7 +331,7 @@
self.log.debug("Running query %s to get project open changes" % (query,))
data = self.gerrit.simpleQuery(query)
changes = []
- for record in data[:-1]:
+ for record in data:
try:
changes.append(self._getChange(record['number'],
record['currentPatchSet']['number']))
@@ -412,3 +413,14 @@
if sha:
url += ';a=commitdiff;h=' + sha
return url
+
+
+def validate_trigger(trigger_data):
+ """Validates the layout's trigger data."""
+ events_with_ref = ('ref-updated', )
+ for event in trigger_data['gerrit']:
+ if event['event'] not in events_with_ref and event.get('ref', False):
+ raise voluptuous.Invalid(
+ "The event %s does not include ref information, Zuul cannot "
+ "use ref filter 'ref: %s'" % (event['event'], event['ref']))
+