Merge "Don't reload connections on HUP"
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index c63700a..9df44ce 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -490,10 +490,12 @@
$header_div.append($heading);
if (typeof pipeline.description === 'string') {
+ var descr = $('<small />')
+ $.each( pipeline.description.split(/\r?\n\r?\n/), function(index, descr_part){
+ descr.append($('<p />').text(descr_part));
+ });
$header_div.append(
- $('<p />').append(
- $('<small />').text(pipeline.description)
- )
+ $('<p />').append(descr)
);
}
return $header_div;
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index b585fea..fe7c7cc 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2235,6 +2235,9 @@
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
+ self.worker.release('project-merge')
+ self.waitUntilSettled()
+
port = self.webapp.server.socket.getsockname()[1]
req = urllib2.Request("http://localhost:%s/status.json" % port)
@@ -2255,7 +2258,7 @@
self.waitUntilSettled()
data = json.loads(data)
- status_jobs = set()
+ status_jobs = []
for p in data['pipelines']:
for q in p['change_queues']:
if p['name'] in ['gate', 'conflict']:
@@ -2267,10 +2270,24 @@
self.assertTrue(change['active'])
self.assertEqual(change['id'], '1,1')
for job in change['jobs']:
- status_jobs.add(job['name'])
- self.assertIn('project-merge', status_jobs)
- self.assertIn('project-test1', status_jobs)
- self.assertIn('project-test2', status_jobs)
+ status_jobs.append(job)
+ self.assertEqual('project-merge', status_jobs[0]['name'])
+ self.assertEqual('https://server/job/project-merge/0/',
+ status_jobs[0]['url'])
+ self.assertEqual('http://logs.example.com/1/1/gate/project-merge/0',
+ status_jobs[0]['report_url'])
+
+ self.assertEqual('project-test1', status_jobs[1]['name'])
+ self.assertEqual('https://server/job/project-test1/1/',
+ status_jobs[1]['url'])
+ self.assertEqual('http://logs.example.com/1/1/gate/project-test1/1',
+ status_jobs[1]['report_url'])
+
+ self.assertEqual('project-test2', status_jobs[2]['name'])
+ self.assertEqual('https://server/job/project-test2/2/',
+ status_jobs[2]['url'])
+ self.assertEqual('http://logs.example.com/1/1/gate/project-test2/2',
+ status_jobs[2]['report_url'])
def test_merging_queues(self):
"Test that transitively-connected change queues are merged"
@@ -4215,6 +4232,45 @@
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
+ def test_crd_cycle_join(self):
+ "Test an updated change creates a cycle"
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Create B->A
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Update A to add A->B (a cycle).
+ A.addPatchset()
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+ # Normally we would submit the patchset-created event for
+ # processing here, however, we have no way of noting whether
+ # the dependency cycle detection correctly raised an
+ # exception, so instead, we reach into the source driver and
+ # call the method that would ultimately be called by the event
+ # processing.
+
+ source = self.sched.layout.pipelines['gate'].source
+ with testtools.ExpectedException(
+ Exception, "Dependency cycle detected"):
+ source._getChange(u'1', u'2', True)
+ self.log.debug("Got expected dependency cycle exception")
+
+ # Now if we update B to remove the depends-on, everything
+ # should be okay. B; A->B
+
+ B.addPatchset()
+ B.data['commitMessage'] = '%s\n' % (B.subject,)
+ source._getChange(u'1', u'2', True)
+ source._getChange(u'2', u'2', True)
+
def test_disable_at(self):
"Test a pipeline will only report to the disabled trigger when failing"
diff --git a/zuul/connection/gerrit.py b/zuul/connection/gerrit.py
index f8e5add..4671ff9 100644
--- a/zuul/connection/gerrit.py
+++ b/zuul/connection/gerrit.py
@@ -94,7 +94,7 @@
try:
event.account = data.get(accountfield_from_type[event.type])
except KeyError:
- self.log.error("Received unrecognized event type '%s' from Gerrit.\
+ self.log.warning("Received unrecognized event type '%s' from Gerrit.\
Can not get account information." % event.type)
event.account = None
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 257b95d..f0235a6 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -103,7 +103,14 @@
repo.fetchFrom(zuul_remote, ref)
self.log.debug("Fetched ref %s from %s", ref, project)
return True
- except (ValueError, GitCommandError):
+ except ValueError:
+ self.log.debug("Project %s in Zuul does not have ref %s",
+ project, ref)
+ return False
+ except GitCommandError as error:
+ # Bail out if fetch fails due to infrastructure reasons
+ if error.stderr.startswith('fatal: unable to access'):
+ raise
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
return False
diff --git a/zuul/model.py b/zuul/model.py
index d2cf13b..5bea5d0 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -266,7 +266,7 @@
items.extend(shared_queue.queue)
return items
- def formatStatusJSON(self):
+ def formatStatusJSON(self, url_pattern=None):
j_pipeline = dict(name=self.name,
description=self.description)
j_queues = []
@@ -283,7 +283,7 @@
if j_changes:
j_queue['heads'].append(j_changes)
j_changes = []
- j_changes.append(e.formatJSON())
+ j_changes.append(e.formatJSON(url_pattern))
if (len(j_changes) > 1 and
(j_changes[-2]['remaining_time'] is not None) and
(j_changes[-1]['remaining_time'] is not None)):
@@ -724,7 +724,34 @@
def setReportedResult(self, result):
self.current_build_set.result = result
- def formatJSON(self):
+ def formatJobResult(self, job, url_pattern=None):
+ build = self.current_build_set.getBuild(job.name)
+ result = build.result
+ pattern = url_pattern
+ if result == 'SUCCESS':
+ if job.success_message:
+ result = job.success_message
+ if job.success_pattern:
+ pattern = job.success_pattern
+ elif result == 'FAILURE':
+ if job.failure_message:
+ result = job.failure_message
+ if job.failure_pattern:
+ pattern = job.failure_pattern
+ url = None
+ if pattern:
+ try:
+ url = pattern.format(change=self.change,
+ pipeline=self.pipeline,
+ job=job,
+ build=build)
+ except Exception:
+ pass # FIXME: log this or something?
+ if not url:
+ url = build.url or job.name
+ return (result, url)
+
+ def formatJSON(self, url_pattern=None):
changeish = self.change
ret = {}
ret['active'] = self.active
@@ -761,11 +788,13 @@
elapsed = None
remaining = None
result = None
- url = None
+ build_url = None
+ report_url = None
worker = None
if build:
result = build.result
- url = build.url
+ build_url = build.url
+ (unused, report_url) = self.formatJobResult(job, url_pattern)
if build.start_time:
if build.end_time:
elapsed = int((build.end_time -
@@ -793,7 +822,8 @@
'name': job.name,
'elapsed_time': elapsed,
'remaining_time': remaining,
- 'url': url,
+ 'url': build_url,
+ 'report_url': report_url,
'result': result,
'voting': job.voting,
'uuid': build.uuid if build else None,
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index fd79174..0569fbe 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -113,25 +113,7 @@
for job in pipeline.getJobs(item):
build = item.current_build_set.getBuild(job.name)
- result = build.result
- pattern = url_pattern
- if result == 'SUCCESS':
- if job.success_message:
- result = job.success_message
- if job.success_pattern:
- pattern = job.success_pattern
- elif result == 'FAILURE':
- if job.failure_message:
- result = job.failure_message
- if job.failure_pattern:
- pattern = job.failure_pattern
- if pattern:
- url = pattern.format(change=item.change,
- pipeline=pipeline,
- job=job,
- build=build)
- else:
- url = build.url or job.name
+ (result, url) = item.formatJobResult(job, url_pattern)
if not job.voting:
voting = ' (non-voting)'
else:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 118cbfc..aea9a67 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -1097,6 +1097,11 @@
pipeline.manager.onMergeCompleted(event)
def formatStatusJSON(self):
+ if self.config.has_option('zuul', 'url_pattern'):
+ url_pattern = self.config.get('zuul', 'url_pattern')
+ else:
+ url_pattern = None
+
data = {}
data['zuul_version'] = self.zuul_version
@@ -1122,7 +1127,7 @@
pipelines = []
data['pipelines'] = pipelines
for pipeline in self.layout.pipelines.values():
- pipelines.append(pipeline.formatStatusJSON())
+ pipelines.append(pipeline.formatStatusJSON(url_pattern))
return json.dumps(data)
@@ -1450,7 +1455,6 @@
return True
if build_set.merge_state == build_set.PENDING:
return False
- build_set.merge_state = build_set.PENDING
ref = build_set.ref
if hasattr(item.change, 'refspec') and not ref:
self.log.debug("Preparing ref for: %s" % item.change)
@@ -1468,6 +1472,8 @@
self.sched.merger.updateRepo(item.change.project.name,
url, build_set,
self.pipeline.precedence)
+ # merge:merge has been emitted properly:
+ build_set.merge_state = build_set.PENDING
return False
def _launchJobs(self, item, jobs):
diff --git a/zuul/source/gerrit.py b/zuul/source/gerrit.py
index eb8705d..73cf726 100644
--- a/zuul/source/gerrit.py
+++ b/zuul/source/gerrit.py
@@ -20,6 +20,20 @@
from zuul.source import BaseSource
+# Walk the change dependency tree to find a cycle
+def detect_cycle(change, history=None):
+ if history is None:
+ history = []
+ else:
+ history = history[:]
+ history.append(change.number)
+ for dep in change.needs_changes:
+ if dep.number in history:
+ raise Exception("Dependency cycle detected: %s in %s" % (
+ dep.number, history))
+ detect_cycle(dep, history)
+
+
class GerritSource(BaseSource):
name = 'gerrit'
log = logging.getLogger("zuul.source.Gerrit")
@@ -60,6 +74,10 @@
data = self.connection.query(change.number)
change._data = data
change.is_merged = self._isMerged(change)
+ if change.is_merged:
+ self.log.debug("Change %s is merged" % (change,))
+ else:
+ self.log.debug("Change %s is not merged" % (change,))
if not head:
return change.is_merged
if not change.is_merged:
@@ -82,7 +100,6 @@
status = data.get('status')
if not status:
return False
- self.log.debug("Change %s status: %s" % (change, status))
if status == 'MERGED':
return True
return False
@@ -177,7 +194,7 @@
(record.get('number'),))
return changes
- def _getDependsOnFromCommit(self, message):
+ def _getDependsOnFromCommit(self, message, change):
records = []
seen = set()
for match in self.depends_on_re.findall(message):
@@ -187,17 +204,19 @@
continue
seen.add(match)
query = "change:%s" % (match,)
- self.log.debug("Running query %s to find needed changes" %
- (query,))
+ self.log.debug("Updating %s: Running query %s "
+ "to find needed changes" %
+ (change, query,))
records.extend(self.connection.simpleQuery(query))
return records
- def _getNeededByFromCommit(self, change_id):
+ def _getNeededByFromCommit(self, change_id, change):
records = []
seen = set()
query = 'message:%s' % change_id
- self.log.debug("Running query %s to find changes needed-by" %
- (query,))
+ self.log.debug("Updating %s: Running query %s "
+ "to find changes needed-by" %
+ (change, query,))
results = self.connection.simpleQuery(query)
for result in results:
for match in self.depends_on_re.findall(
@@ -207,15 +226,15 @@
key = (result['number'], result['currentPatchSet']['number'])
if key in seen:
continue
- self.log.debug("Found change %s,%s needs %s from commit" %
- (key[0], key[1], change_id))
+ self.log.debug("Updating %s: Found change %s,%s "
+ "needs %s from commit" %
+ (change, key[0], key[1], change_id))
seen.add(key)
records.append(result)
return records
def _updateChange(self, change, history=None):
- self.log.info("Updating information for %s,%s" %
- (change.number, change.patchset))
+ self.log.info("Updating %s" % (change,))
data = self.connection.query(change.number)
change._data = data
@@ -255,6 +274,7 @@
if change.is_merged:
# This change is merged, so we don't need to look any further
# for dependencies.
+ self.log.debug("Updating %s: change is merged" % (change,))
return change
if history is None:
@@ -270,21 +290,35 @@
if dep_num in history:
raise Exception("Dependency cycle detected: %s in %s" % (
dep_num, history))
- self.log.debug("Getting git-dependent change %s,%s" %
- (dep_num, dep_ps))
+ self.log.debug("Updating %s: Getting git-dependent change %s,%s" %
+ (change, dep_num, dep_ps))
dep = self._getChange(dep_num, dep_ps, history=history)
+ # Because we are not forcing a refresh in _getChange, it
+ # may return without executing this code, so if we are
+ # updating our change to add ourselves to a dependency
+ # cycle, we won't detect it. By explicitly performing a
+ # walk of the dependency tree, we will.
+ detect_cycle(dep, history)
if (not dep.is_merged) and dep not in needs_changes:
needs_changes.append(dep)
- for record in self._getDependsOnFromCommit(data['commitMessage']):
+ for record in self._getDependsOnFromCommit(data['commitMessage'],
+ change):
dep_num = record['number']
dep_ps = record['currentPatchSet']['number']
if dep_num in history:
raise Exception("Dependency cycle detected: %s in %s" % (
dep_num, history))
- self.log.debug("Getting commit-dependent change %s,%s" %
- (dep_num, dep_ps))
+ self.log.debug("Updating %s: Getting commit-dependent "
+ "change %s,%s" %
+ (change, dep_num, dep_ps))
dep = self._getChange(dep_num, dep_ps, history=history)
+ # Because we are not forcing a refresh in _getChange, it
+ # may return without executing this code, so if we are
+ # updating our change to add ourselves to a dependency
+ # cycle, we won't detect it. By explicitly performing a
+ # walk of the dependency tree, we will.
+ detect_cycle(dep, history)
if (not dep.is_merged) and dep not in needs_changes:
needs_changes.append(dep)
change.needs_changes = needs_changes
@@ -294,15 +328,17 @@
for needed in data['neededBy']:
parts = needed['ref'].split('/')
dep_num, dep_ps = parts[3], parts[4]
+ self.log.debug("Updating %s: Getting git-needed change %s,%s" %
+ (change, dep_num, dep_ps))
dep = self._getChange(dep_num, dep_ps)
if (not dep.is_merged) and dep.is_current_patchset:
needed_by_changes.append(dep)
- for record in self._getNeededByFromCommit(data['id']):
+ for record in self._getNeededByFromCommit(data['id'], change):
dep_num = record['number']
dep_ps = record['currentPatchSet']['number']
- self.log.debug("Getting commit-needed change %s,%s" %
- (dep_num, dep_ps))
+ self.log.debug("Updating %s: Getting commit-needed change %s,%s" %
+ (change, dep_num, dep_ps))
# Because a commit needed-by may be a cross-repo
# dependency, cause that change to refresh so that it will
# reference the latest patchset of its Depends-On (this