Merge "Add gerrit reviews into patchset approvals"
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index 6c77477..7c3e384 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -635,7 +635,7 @@
Default: ``linear``.
**window-increase-factor**
- DependentPipelineManagers only. The value to be added or mulitplied
+ DependentPipelineManagers only. The value to be added or multiplied
against the previous window value to determine the new window after
successful change merges.
Default: ``1``.
diff --git a/etc/status/.jshintignore b/etc/status/.jshintignore
new file mode 100644
index 0000000..218f297
--- /dev/null
+++ b/etc/status/.jshintignore
@@ -0,0 +1 @@
+public_html/lib
diff --git a/etc/status/.jshintrc b/etc/status/.jshintrc
new file mode 100644
index 0000000..15bd571
--- /dev/null
+++ b/etc/status/.jshintrc
@@ -0,0 +1,21 @@
+{
+ "bitwise": true,
+ "eqeqeq": true,
+ "forin": true,
+ "latedef": true,
+ "newcap": true,
+ "noarg": true,
+ "noempty": true,
+ "nonew": true,
+ "undef": true,
+ "unused": true,
+
+ "strict": false,
+ "laxbreak": true,
+ "browser": true,
+
+ "predef": [
+ "jQuery",
+ "zuul"
+ ]
+}
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index 40a5d4d..1db3c8e 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -16,9 +16,10 @@
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
-'use strict';
(function ($) {
+ 'use strict';
+
function set_cookie(name, value) {
document.cookie = name + '=' + value + '; path=/';
}
@@ -39,7 +40,7 @@
}
$.zuul = function(options) {
- var options = $.extend({
+ options = $.extend({
'enabled': true,
'graphite_url': '',
'source': 'status.json',
@@ -72,7 +73,7 @@
hideGrid: true,
target: [
"color(stats.gauges.zuul.pipeline." + pipeline_name
- + ".current_changes, '6b8182')"
+ + ".current_changes, '6b8182')"
]
});
}
@@ -616,7 +617,7 @@
var app = {
schedule: function (app) {
- var app = app || this;
+ app = app || this;
if (!options.enabled) {
setTimeout(function() {app.schedule(app);}, 5000);
return;
@@ -642,7 +643,7 @@
this.emit('update-start');
var app = this;
- var $msg = $(options.msg_id)
+ var $msg = $(options.msg_id);
xhr = $.getJSON(options.source)
.done(function (data) {
if ('message' in data) {
@@ -682,7 +683,10 @@
data.result_event_queue.length : '0'
);
})
- .fail(function (err, jqXHR, errMsg) {
+ .fail(function (jqXHR, statusText, errMsg) {
+ if (statusText === 'abort') {
+ return;
+ }
$msg.text(options.source + ': ' + errMsg)
.addClass('alert-danger')
.removeClass('zuul-msg-wrap-off')
@@ -701,7 +705,7 @@
var newimg = new Image();
var parts = url.split('#');
newimg.src = parts[0] + '#' + new Date().getTime();
- $(newimg).load(function (x) {
+ $(newimg).load(function () {
zuul_sparkline_urls[name] = newimg.src;
});
});
@@ -897,5 +901,5 @@
app: app,
jq: $jq
};
- }
+ };
}(jQuery));
diff --git a/etc/status/public_html/zuul.app.js b/etc/status/public_html/zuul.app.js
index 640437b..6321af8 100644
--- a/etc/status/public_html/zuul.app.js
+++ b/etc/status/public_html/zuul.app.js
@@ -17,9 +17,11 @@
// License for the specific language governing permissions and limitations
// under the License.
+/*exported zuul_build_dom, zuul_start */
+
function zuul_build_dom($, container) {
// Build a default-looking DOM
- default_layout = '<div class="container">'
+ var default_layout = '<div class="container">'
+ '<h1>Zuul Status</h1>'
+ '<p>Real-time status monitor of Zuul, the pipeline manager between Gerrit and Workers.</p>'
+ '<div class="zuul-container" id="zuul-container">'
@@ -34,7 +36,7 @@
$(function ($) {
// DOM ready
- $container = $(container);
+ var $container = $(container);
$container.html(default_layout);
});
}
diff --git a/tests/base.py b/tests/base.py
index d324113..5ddb160 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -47,8 +47,9 @@
import zuul.rpclistener
import zuul.launcher.gearman
import zuul.lib.swift
-import zuul.merger.server
import zuul.merger.client
+import zuul.merger.merger
+import zuul.merger.server
import zuul.reporter.gerrit
import zuul.reporter.smtp
import zuul.trigger.gerrit
@@ -145,7 +146,7 @@
self.latest_patchset),
'refs/tags/init')
repo.head.reference = ref
- repo.head.reset(index=True, working_tree=True)
+ zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
path = os.path.join(self.upstream_root, self.project)
@@ -167,7 +168,7 @@
r = repo.index.commit(msg)
repo.head.reference = 'master'
- repo.head.reset(index=True, working_tree=True)
+ zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
repo.heads['master'].checkout()
return r
@@ -917,6 +918,7 @@
self.init_repo("org/conflict-project")
self.init_repo("org/noop-project")
self.init_repo("org/experimental-project")
+ self.init_repo("org/no-jobs-project")
self.statsd = FakeStatsd()
os.environ['STATSD_HOST'] = 'localhost'
@@ -1081,7 +1083,7 @@
repo.create_tag('init')
repo.head.reference = master
- repo.head.reset(index=True, working_tree=True)
+ zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
self.create_branch(project, 'mp')
@@ -1100,7 +1102,7 @@
repo.index.commit('%s commit' % branch)
repo.head.reference = repo.heads['master']
- repo.head.reset(index=True, working_tree=True)
+ zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
def ref_has_change(self, ref, change):
diff --git a/tests/fixtures/layout-live-reconfiguration-add-job.yaml b/tests/fixtures/layout-live-reconfiguration-add-job.yaml
new file mode 100644
index 0000000..e4aea6f
--- /dev/null
+++ b/tests/fixtures/layout-live-reconfiguration-add-job.yaml
@@ -0,0 +1,38 @@
+pipelines:
+ - name: gate
+ manager: DependentPipelineManager
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+jobs:
+ - name: ^.*-merge$
+ failure-message: Unable to merge change
+ hold-following-changes: true
+ - name: project-testfile
+ files:
+ - '.*-requires'
+
+projects:
+ - name: org/project
+ merge-mode: cherry-pick
+ gate:
+ - project-merge:
+ - project-test1
+ - project-test2
+ - project-test3
+ - project-testfile
diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml
index cc4d34c..1d23443 100644
--- a/tests/fixtures/layout.yaml
+++ b/tests/fixtures/layout.yaml
@@ -246,3 +246,7 @@
- name: org/experimental-project
experimental:
- experimental-project-test
+
+ - name: org/no-jobs-project
+ check:
+ - project-testfile
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 3b59e3e..5b9a39d 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1878,6 +1878,23 @@
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
+ def test_no_job_project(self):
+ "Test that reports with no jobs don't get sent"
+ A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
+ 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Change wasn't reported to
+ self.assertEqual(A.reported, False)
+
+ # Check queue is empty afterwards
+ check_pipeline = self.sched.layout.pipelines['check']
+ items = check_pipeline.getAllItems()
+ self.assertEqual(len(items), 0)
+
+ self.assertEqual(len(self.history), 0)
+
def test_zuul_refs(self):
"Test that zuul refs exist and have the right changes"
self.worker.hold_jobs_in_build = True
@@ -2019,6 +2036,30 @@
self.assertEqual(self.history[0].name, 'gate-noop')
self.assertEqual(self.history[0].result, 'SUCCESS')
+ def test_file_head(self):
+ # This is a regression test for an observed bug. A change
+ # with a file named "HEAD" in the root directory of the repo
+ # was processed by a merger. It then was unable to reset the
+ # repo because of:
+ # GitCommandError: 'git reset --hard HEAD' returned
+ # with exit code 128
+ # stderr: 'fatal: ambiguous argument 'HEAD': both revision
+ # and filename
+ # Use '--' to separate filenames from revisions'
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addPatchset(['HEAD'])
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertIn('Build succeeded', A.messages[0])
+ self.assertIn('Build succeeded', B.messages[0])
+
def test_file_jobs(self):
"Test that file jobs run only when appropriate"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
@@ -2224,6 +2265,127 @@
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
+ def test_live_reconfiguration_merge_conflict(self):
+ # A real-world bug: a change in a gate queue has a merge
+ # conflict and a job is added to its project while it's
+ # sitting in the queue. The job gets added to the change and
+ # enqueued and the change gets stuck.
+ self.worker.registerFunction('build:project-test3')
+ self.worker.hold_jobs_in_build = True
+
+ # This change is fine. It's here to stop the queue long
+ # enough for the next change to be subject to the
+ # reconfiguration, as well as to provide a conflict for the
+ # next change. This change will succeed and merge.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addPatchset(['conflict'])
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+
+ # This change will be in merge conflict. During the
+ # reconfiguration, we will add a job. We want to make sure
+ # that doesn't cause it to get stuck.
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ B.addPatchset(['conflict'])
+ B.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+
+ self.waitUntilSettled()
+
+ # No jobs have run yet
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 1)
+ self.assertEqual(len(self.history), 0)
+
+ # Add the "project-test3" job.
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-live-'
+ 'reconfiguration-add-job.yaml')
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(self.getJobFromHistory('project-merge').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('project-test2').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('project-test3').result,
+ 'SUCCESS')
+ self.assertEqual(len(self.history), 4)
+
+ def test_live_reconfiguration_failed_job(self):
+ # An extrapolation of test_live_reconfiguration_merge_conflict
+ # that tests a job added to a job tree with a failed root does
+ # not run.
+ self.worker.registerFunction('build:project-test3')
+ self.worker.hold_jobs_in_build = True
+
+ # This change is fine. It's here to stop the queue long
+ # enough for the next change to be subject to the
+ # reconfiguration. This change will succeed and merge.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addPatchset(['conflict'])
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.worker.release('.*-merge')
+ self.waitUntilSettled()
+
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ self.worker.addFailTest('project-merge', B)
+ B.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+ self.waitUntilSettled()
+
+ self.worker.release('.*-merge')
+ self.waitUntilSettled()
+
+ # Both -merge jobs have run, but no others.
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 1)
+ self.assertEqual(self.history[0].result, 'SUCCESS')
+ self.assertEqual(self.history[0].name, 'project-merge')
+ self.assertEqual(self.history[1].result, 'FAILURE')
+ self.assertEqual(self.history[1].name, 'project-merge')
+ self.assertEqual(len(self.history), 2)
+
+ # Add the "project-test3" job.
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-live-'
+ 'reconfiguration-add-job.yaml')
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(self.history[0].result, 'SUCCESS')
+ self.assertEqual(self.history[0].name, 'project-merge')
+ self.assertEqual(self.history[1].result, 'FAILURE')
+ self.assertEqual(self.history[1].name, 'project-merge')
+ self.assertEqual(self.history[2].result, 'SUCCESS')
+ self.assertEqual(self.history[3].result, 'SUCCESS')
+ self.assertEqual(self.history[4].result, 'SUCCESS')
+ self.assertEqual(len(self.history), 5)
+
def test_live_reconfiguration_functions(self):
"Test live reconfiguration with a custom function"
self.worker.registerFunction('build:node-project-test1:debian')
diff --git a/zuul/launcher/gearman.py b/zuul/launcher/gearman.py
index 653678a..57ac5ca 100644
--- a/zuul/launcher/gearman.py
+++ b/zuul/launcher/gearman.py
@@ -265,12 +265,14 @@
params))
def launch(self, job, item, pipeline, dependent_items=[]):
- self.log.info("Launch job %s for change %s with dependent changes %s" %
- (job, item.change,
- [x.change for x in dependent_items]))
+ uuid = str(uuid4().hex)
+ self.log.info(
+ "Launch job %s (uuid: %s) for change %s with dependent "
+ "changes %s" % (
+ job, uuid, item.change,
+ [x.change for x in dependent_items]))
dependent_items = dependent_items[:]
dependent_items.reverse()
- uuid = str(uuid4().hex)
params = dict(ZUUL_UUID=uuid,
ZUUL_PROJECT=item.change.project.name)
params['ZUUL_PIPELINE'] = pipeline.name
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index f36b974..f571ad0 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -20,6 +20,21 @@
import zuul.model
+def reset_repo_to_head(repo):
+ # This lets us reset the repo even if there is a file in the root
+ # directory named 'HEAD'. Currently, GitPython does not allow us
+ # to instruct it to always include the '--' to disambiguate. This
+ # should no longer be necessary if this PR merges:
+ # https://github.com/gitpython-developers/GitPython/pull/319
+ try:
+ repo.git.reset('--hard', 'HEAD', '--')
+ except git.GitCommandError as e:
+ # git nowadays may use 1 as status to indicate there are still unstaged
+ # modifications after the reset
+ if e.status != 1:
+ raise
+
+
class ZuulReference(git.Reference):
_common_path_default = "refs/zuul"
_points_to_commits_only = True
@@ -82,7 +97,7 @@
# Reset to remote HEAD (usually origin/master)
repo.head.reference = origin.refs['HEAD']
- repo.head.reset(index=True, working_tree=True)
+ reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
def prune(self):
@@ -114,7 +129,7 @@
repo = self.createRepoObject()
self.log.debug("Checking out %s" % ref)
repo.head.reference = ref
- repo.head.reset(index=True, working_tree=True)
+ reset_repo_to_head(repo)
def cherryPick(self, ref):
repo = self.createRepoObject()
diff --git a/zuul/model.py b/zuul/model.py
index 4d402ff..4b907c3 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -735,7 +735,13 @@
ret['items_behind'] = [i.change._id() for i in self.items_behind]
ret['failing_reasons'] = self.current_build_set.failing_reasons
ret['zuul_ref'] = self.current_build_set.ref
- ret['project'] = changeish.project.name
+ if changeish.project:
+ ret['project'] = changeish.project.name
+ else:
+ # For cross-project dependencies with the depends-on
+ # project not known to zuul, the project is None
+ # Set it to a static value
+ ret['project'] = "Unknown Project"
ret['enqueue_time'] = int(self.enqueue_time * 1000)
ret['jobs'] = []
if hasattr(changeish, 'owner'):
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 131ad62..58e8a96 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -198,7 +198,7 @@
self.management_event_queue = Queue.Queue()
self.layout = model.Layout()
- self.zuul_version = zuul_version.version_info.version_string()
+ self.zuul_version = zuul_version.version_info.release_string()
self.last_reconfigured = None
def stop(self):
@@ -1176,6 +1176,18 @@
self.log.debug("Re-enqueing change %s in queue %s" %
(item.change, change_queue))
change_queue.enqueueItem(item)
+
+ # Re-set build results in case any new jobs have been
+ # added to the tree.
+ for build in item.current_build_set.getBuilds():
+ if build.result:
+ self.pipeline.setResult(item, build)
+ # Similarly, reset the item state.
+ if item.current_build_set.unable_to_merge:
+ self.pipeline.setUnableToMerge(item)
+ if item.dequeued_needing_change:
+ self.pipeline.setDequeuedNeedingChange(item)
+
self.reportStats(item)
return True
else:
@@ -1521,7 +1533,13 @@
def _reportItem(self, item):
self.log.debug("Reporting change %s" % item.change)
ret = True # Means error as returned by trigger.report
- if self.pipeline.didAllJobsSucceed(item):
+ if not self.pipeline.getJobs(item):
+ # We don't send empty reports with +1,
+ # and the same for -1's (merge failures or transient errors)
+ # as they cannot be followed by +1's
+ self.log.debug("No jobs for change %s" % item.change)
+ actions = []
+ elif self.pipeline.didAllJobsSucceed(item):
self.log.debug("success %s" % (self.pipeline.success_actions))
actions = self.pipeline.success_actions
item.setReportedResult('SUCCESS')