Merge "Support pagination in simplequery for Gerrit >=2.9"
diff --git a/doc/source/cloner.rst b/doc/source/cloner.rst
index 2ddf0b5..70577cc 100644
--- a/doc/source/cloner.rst
+++ b/doc/source/cloner.rst
@@ -61,6 +61,23 @@
.. program-output:: zuul-cloner --help
+
+Ref lookup order
+''''''''''''''''
+
+The Zuul cloner will attempt to lookup references in this order:
+
+ 1) Zuul reference for the indicated branch
+ 2) Zuul reference for the master branch
+ 3) The tip of the indicated branch
+ 4) The tip of the master branch
+
+The "indicated branch" is one of the following:
+
+ A) The project-specific override branch (from project_branches arg)
+ B) The user specified branch (from the branch arg)
+ C) ZUUL_BRANCH (from the zuul_branch arg)
+
Clone order
-----------
@@ -87,3 +104,7 @@
The URL of origin remote of the resulting clone will be reset to use
the ``git_base_url`` and then the remote will be updated so that the
repository has all the information in the upstream repository.
+
+The default for ``--cache-dir`` is taken from the environment variable
+``ZUUL_CACHE_DIR``. A value provided explicitly on the command line
+overrides the environment variable setting.
diff --git a/doc/source/gating.rst b/doc/source/gating.rst
index 43a5928..fff2924 100644
--- a/doc/source/gating.rst
+++ b/doc/source/gating.rst
@@ -212,19 +212,20 @@
}
-Cross projects dependencies
----------------------------
+Cross Project Testing
+---------------------
When your projects are closely coupled together, you want to make sure
changes entering the gate are going to be tested with the version of
other projects currently enqueued in the gate (since they will
eventually be merged and might introduce breaking features).
-Such dependencies can be defined in Zuul configuration by registering a job
-in a DependentPipeline of several projects. Whenever a change enters such a
-pipeline, it will create references for the other projects as well. As an
-example, given a main project ``acme`` and a plugin ``plugin`` you can
-define a job ``acme-tests`` which should be run for both projects:
+Such relationships can be defined in Zuul configuration by registering
+a job in a DependentPipeline of several projects. Whenever a change
+enters such a pipeline, it will create references for the other
+projects as well. As an example, given a main project ``acme`` and a
+plugin ``plugin`` you can define a job ``acme-tests`` which should be
+run for both projects:
.. code-block:: yaml
@@ -280,3 +281,82 @@
When your job fetches several repositories without changes ahead in the
queue, they may not have a Z reference in which case you can just check
out the branch.
+
+
+Cross Repository Dependencies
+-----------------------------
+
+Zuul permits users to specify dependencies across repositories. Using
+a special header in Git commit messages, Users may specify that a
+change depends on another change in any repository known to Zuul.
+
+Zuul's cross-repository dependencies (CRD) behave like a directed
+acyclic graph (DAG), like git itself, to indicate a one-way dependency
+relationship between changes in different git repositories. Change A
+may depend on B, but B may not depend on A.
+
+To use them, include "Depends-On: <gerrit-change-id>" in the footer of
+a commit message. Use the full Change-ID ('I' + 40 characters).
+
+
+Gate Pipeline
+~~~~~~~~~~~~~
+
+When Zuul sees CRD changes, it serializes them in the usual manner when
+enqueuing them into a pipeline. This means that if change A depends on
+B, then when they are added to the gate pipeline, B will appear first
+and A will follow. If tests for B fail, both B and A will be removed
+from the pipeline, and it will not be possible for A to merge until B
+does.
+
+Note that if changes with CRD do not share a change queue then Zuul
+is unable to enqueue them together, and the first will be required to
+merge before the second is enqueued.
+
+Check Pipeline
+~~~~~~~~~~~~~~
+
+When changes are enqueued into the check pipeline, all of the related
+dependencies (both normal git-dependencies that come from parent commits
+as well as CRD changes) appear in a dependency graph, as in gate. This
+means that even in the check pipeline, your change will be tested with
+its dependency. So changes that were previously unable to be fully
+tested until a related change landed in a different repo may now be
+tested together from the start.
+
+All of the changes are still independent (so you will note that the
+whole pipeline does not share a graph as in gate), but for each change
+tested, all of its dependencies are visually connected to it, and they
+are used to construct the git references that Zuul uses when testing.
+When looking at this graph on the status page, you will note that the
+dependencies show up as grey dots, while the actual change tested shows
+up as red or green. This is to indicate that the grey changes are only
+there to establish dependencies. Even if one of the dependencies is
+also being tested, it will show up as a grey dot when used as a
+dependency, but separately and additionally will appear as its own red
+or green dot for its test.
+
+Multiple Changes
+~~~~~~~~~~~~~~~~
+
+A Gerrit change ID may refer to multiple changes (on multiple branches
+of the same project, or even multiple projects). In these cases, Zuul
+will treat all of the changes with that change ID as dependencies. So
+if you say that change in project A Depends-On a change ID that has
+changes in two branches of project B, then when testing the change to
+project A, both project B changes will be applied, and when deciding
+whether the project A change can merge, both changes must merge ahead
+of it.
+
+A change may depend on more than one Gerrit change ID as well. So it
+is possible for a change in project A to depend on a change in project
+B and a change in project C. Simply add more "Depends-On:" lines to
+the footer.
+
+Cycles
+~~~~~~
+
+If a cycle is created by use of CRD, Zuul will abort its work very
+early. There will be no message in Gerrit and no changes that are part
+of the cycle will be enqueued into any pipeline. This is to protect
+Zuul from infinite loops.
diff --git a/doc/source/launchers.rst b/doc/source/launchers.rst
index b95354f..0a1e0e7 100644
--- a/doc/source/launchers.rst
+++ b/doc/source/launchers.rst
@@ -66,6 +66,11 @@
**LOG_PATH**
zuul also suggests a unique path for logs to the worker. This is
"BASE_LOG_PATH/pipeline-name/job-name/uuid"
+**ZUUL_VOTING**
+ Whether Zuul considers this job voting or not. Note that if Zuul is
+ reconfigured during the run, the voting status of a job may change
+ and this value will be out of date. Values are '1' if voting, '0'
+ otherwise.
Change related parameters
~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index bcdfabb..d597f23 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -426,6 +426,12 @@
provides. This field is treated as a regular expression, and
multiple refs may be listed.
+ *ignore-deletes*
+ When a branch is deleted, a ref-updated event is emitted with a newrev
+ of all zeros specified. The ``ignore-deletes`` field is a boolean value
+ that describes whether or not these newrevs trigger ref-updated events.
+ The default is True, which will not trigger ref-updated events.
+
*approval*
This is only used for ``comment-added`` events. It only matches if
the event has a matching approval associated with it. Example:
@@ -607,6 +613,18 @@
do when a change is added to the pipeline manager. This can be used,
for example, to reset the value of the Verified review category.
+**disabled**
+ Uses the same syntax as **success**, but describes what Zuul should
+ do when a pipeline is disabled.
+ See ``disable-after-consecutive-failures``.
+
+**disable-after-consecutive-failures**
+ If set, a pipeline can enter a ''disabled'' state if too many changes
+ in a row fail. When this value is exceeded the pipeline will stop
+ reporting to any of the ``success``, ``failure`` or ``merge-failure``
+ reporters and instead only report to the ``disabled`` reporters.
+ (No ``start`` reports are made when a pipeline is disabled).
+
**precedence**
Indicates how the build scheduler should prioritize jobs for
different pipelines. Each pipeline may have one precedence, jobs
diff --git a/etc/layout.yaml-sample b/etc/layout.yaml-sample
index 30a3352..53f6ba1 100644
--- a/etc/layout.yaml-sample
+++ b/etc/layout.yaml-sample
@@ -30,6 +30,7 @@
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
+ ignore-deletes: False
- name: gate
manager: DependentPipelineManager
diff --git a/tests/base.py b/tests/base.py
index 5ddb160..abbdb0a 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -624,6 +624,8 @@
result = 'RUN_ERROR'
else:
data['result'] = result
+ data['node_labels'] = ['bare-necessities']
+ data['node_name'] = 'foo'
work_fail = False
changes = None
diff --git a/tests/fixtures/layout-disable-at.yaml b/tests/fixtures/layout-disable-at.yaml
new file mode 100644
index 0000000..a2b2526
--- /dev/null
+++ b/tests/fixtures/layout-disable-at.yaml
@@ -0,0 +1,21 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+ disabled:
+ smtp:
+ to: you@example.com
+ disable-after-consecutive-failures: 3
+
+projects:
+ - name: org/project
+ check:
+ - project-test1
diff --git a/tests/fixtures/layout-dont-ignore-deletes.yaml b/tests/fixtures/layout-dont-ignore-deletes.yaml
new file mode 100644
index 0000000..1cf3c71
--- /dev/null
+++ b/tests/fixtures/layout-dont-ignore-deletes.yaml
@@ -0,0 +1,16 @@
+includes:
+ - python-file: custom_functions.py
+
+pipelines:
+ - name: post
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: ref-updated
+ ref: ^(?!refs/).*$
+ ignore-deletes: False
+
+projects:
+ - name: org/project
+ post:
+ - project-post
diff --git a/tests/fixtures/layout-live-reconfiguration-del-project.yaml b/tests/fixtures/layout-live-reconfiguration-del-project.yaml
new file mode 100644
index 0000000..07ffb2e
--- /dev/null
+++ b/tests/fixtures/layout-live-reconfiguration-del-project.yaml
@@ -0,0 +1,21 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+projects:
+ - name: org/project
+ merge-mode: cherry-pick
+ check:
+ - project-merge:
+ - project-test1
+ - project-test2
+ - project-testfile
diff --git a/tests/fixtures/layouts/good_layout.yaml b/tests/fixtures/layouts/good_layout.yaml
index fc2effd..9ba1806 100644
--- a/tests/fixtures/layouts/good_layout.yaml
+++ b/tests/fixtures/layouts/good_layout.yaml
@@ -20,6 +20,7 @@
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
+ ignore-deletes: True
- name: gate
manager: DependentPipelineManager
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 61a2d09..a257440 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -111,6 +111,9 @@
self.assertReportedStat(
'zuul.pipeline.gate.org.project.total_changes', value='1|c')
+ for build in self.builds:
+ self.assertEqual(build.parameters['ZUUL_VOTING'], '1')
+
def test_initial_pipeline_gauges(self):
"Test that each pipeline reported its length on start"
pipeline_names = self.sched.layout.pipelines.keys()
@@ -891,6 +894,54 @@
self.assertEqual(len(self.history), 1)
self.assertIn('project-post', job_names)
+ def test_post_ignore_deletes(self):
+ "Test that deleting refs does not trigger post jobs"
+
+ e = {
+ "type": "ref-updated",
+ "submitter": {
+ "name": "User Name",
+ },
+ "refUpdate": {
+ "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
+ "newRev": "0000000000000000000000000000000000000000",
+ "refName": "master",
+ "project": "org/project",
+ }
+ }
+ self.fake_gerrit.addEvent(e)
+ self.waitUntilSettled()
+
+ job_names = [x.name for x in self.history]
+ self.assertEqual(len(self.history), 0)
+ self.assertNotIn('project-post', job_names)
+
+ def test_post_ignore_deletes_negative(self):
+ "Test that deleting refs does trigger post jobs"
+
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-dont-ignore-deletes.yaml')
+ self.sched.reconfigure(self.config)
+
+ e = {
+ "type": "ref-updated",
+ "submitter": {
+ "name": "User Name",
+ },
+ "refUpdate": {
+ "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
+ "newRev": "0000000000000000000000000000000000000000",
+ "refName": "master",
+ "project": "org/project",
+ }
+ }
+ self.fake_gerrit.addEvent(e)
+ self.waitUntilSettled()
+
+ job_names = [x.name for x in self.history]
+ self.assertEqual(len(self.history), 1)
+ self.assertIn('project-post', job_names)
+
def test_build_configuration_branch(self):
"Test that the right commits are on alternate branches"
@@ -1256,6 +1307,9 @@
self.getJobFromHistory('nonvoting-project-test2').result,
'FAILURE')
+ for build in self.builds:
+ self.assertEqual(build.parameters['ZUUL_VOTING'], '0')
+
def test_check_queue_success(self):
"Test successful check queue jobs."
@@ -2494,6 +2548,57 @@
# Ensure the removed job was not included in the report.
self.assertNotIn('project1-project2-integration', A.messages[0])
+ def test_live_reconfiguration_del_project(self):
+ # Test project deletion from layout
+ # while changes are enqueued
+
+ self.worker.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.worker.release('.*-merge')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 5)
+
+ # This layout defines only org/project, not org/project1
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-live-'
+ 'reconfiguration-del-project.yaml')
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ # Builds for C aborted, builds for A succeed,
+ # and have change B applied ahead
+ job_c = self.getJobFromHistory('project1-test1')
+ self.assertEqual(job_c.changes, '3,1')
+ self.assertEqual(job_c.result, 'ABORTED')
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(self.getJobFromHistory('project-test1').changes,
+ '2,1 1,1')
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(C.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(C.reported, 0)
+
+ self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0)
+ self.assertIn('Build succeeded', A.messages[0])
+
def test_live_reconfiguration_functions(self):
"Test live reconfiguration with a custom function"
self.worker.registerFunction('build:node-project-test1:debian')
@@ -3668,6 +3773,48 @@
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
+ def test_crd_gate_unknown(self):
+ "Test unknown projects in dependent pipeline"
+ self.init_repo("org/unknown")
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
+ A.addApproval('CRVW', 2)
+ B.addApproval('CRVW', 2)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ B.addApproval('APRV', 1)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+
+ # Unknown projects cannot share a queue with any other
+ # since they don't have common jobs with any other (they have no jobs).
+ # Changes which depend on unknown project changes
+ # should not be processed in dependent pipeline
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Simulate change B being gated outside this layout
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+ B.setMerged()
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 0)
+
def test_crd_check(self):
"Test cross-repo dependencies in independent pipelines"
@@ -3782,12 +3929,12 @@
self.assertIn('Build succeeded', A.messages[0])
self.assertIn('Build succeeded', B.messages[0])
- def test_crd_check_reconfiguration(self):
+ def _test_crd_check_reconfiguration(self, project1, project2):
"Test cross-repo dependencies re-enqueued in independent pipelines"
self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
+ B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
@@ -3820,6 +3967,17 @@
self.assertEqual(self.history[0].changes, '2,1 1,1')
self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0)
+ def test_crd_check_reconfiguration(self):
+ self._test_crd_check_reconfiguration('org/project1', 'org/project2')
+
+ def test_crd_undefined_project(self):
+ """Test that undefined projects in dependencies are handled for
+ independent pipelines"""
+ # It's a hack for fake gerrit,
+ # as it implies repo creation upon the creation of any change
+ self.init_repo("org/unknown")
+ self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
+
def test_crd_check_ignore_dependencies(self):
"Test cross-repo dependencies can be ignored"
self.config.set('zuul', 'layout_config',
@@ -3904,3 +4062,125 @@
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
+
+ def test_disable_at(self):
+ "Test a pipeline will only report to the disabled trigger when failing"
+
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-disable-at.yaml')
+ self.sched.reconfigure(self.config)
+
+ self.assertEqual(3, self.sched.layout.pipelines['check'].disable_at)
+ self.assertEqual(
+ 0, self.sched.layout.pipelines['check']._consecutive_failures)
+ self.assertFalse(self.sched.layout.pipelines['check']._disabled)
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
+ D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
+ E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
+ F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
+ G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
+ H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')
+ I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')
+ J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')
+ K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')
+
+ self.worker.addFailTest('project-test1', A)
+ self.worker.addFailTest('project-test1', B)
+ # Let C pass, resetting the counter
+ self.worker.addFailTest('project-test1', D)
+ self.worker.addFailTest('project-test1', E)
+ self.worker.addFailTest('project-test1', F)
+ self.worker.addFailTest('project-test1', G)
+ self.worker.addFailTest('project-test1', H)
+ # I also passes but should only report to the disabled reporters
+ self.worker.addFailTest('project-test1', J)
+ self.worker.addFailTest('project-test1', K)
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(
+ 2, self.sched.layout.pipelines['check']._consecutive_failures)
+ self.assertFalse(self.sched.layout.pipelines['check']._disabled)
+
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(
+ 0, self.sched.layout.pipelines['check']._consecutive_failures)
+ self.assertFalse(self.sched.layout.pipelines['check']._disabled)
+
+ self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # We should be disabled now
+ self.assertEqual(
+ 3, self.sched.layout.pipelines['check']._consecutive_failures)
+ self.assertTrue(self.sched.layout.pipelines['check']._disabled)
+
+ # We need to wait between each of these patches to make sure the
+ # smtp messages come back in an expected order
+ self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # The first 6 (ABCDEF) jobs should have reported back to gerrt thus
+ # leaving a message on each change
+ self.assertEqual(1, len(A.messages))
+ self.assertIn('Build failed.', A.messages[0])
+ self.assertEqual(1, len(B.messages))
+ self.assertIn('Build failed.', B.messages[0])
+ self.assertEqual(1, len(C.messages))
+ self.assertIn('Build succeeded.', C.messages[0])
+ self.assertEqual(1, len(D.messages))
+ self.assertIn('Build failed.', D.messages[0])
+ self.assertEqual(1, len(E.messages))
+ self.assertIn('Build failed.', E.messages[0])
+ self.assertEqual(1, len(F.messages))
+ self.assertIn('Build failed.', F.messages[0])
+
+ # The last 3 (GHI) would have only reported via smtp.
+ self.assertEqual(3, len(self.smtp_messages))
+ self.assertEqual(0, len(G.messages))
+ self.assertIn('Build failed.', self.smtp_messages[0]['body'])
+ self.assertIn('/7/1/check', self.smtp_messages[0]['body'])
+ self.assertEqual(0, len(H.messages))
+ self.assertIn('Build failed.', self.smtp_messages[1]['body'])
+ self.assertIn('/8/1/check', self.smtp_messages[1]['body'])
+ self.assertEqual(0, len(I.messages))
+ self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])
+ self.assertIn('/9/1/check', self.smtp_messages[2]['body'])
+
+ # Now reload the configuration (simulate a HUP) to check the pipeline
+ # comes out of disabled
+ self.sched.reconfigure(self.config)
+
+ self.assertEqual(3, self.sched.layout.pipelines['check'].disable_at)
+ self.assertEqual(
+ 0, self.sched.layout.pipelines['check']._consecutive_failures)
+ self.assertFalse(self.sched.layout.pipelines['check']._disabled)
+
+ self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(
+ 2, self.sched.layout.pipelines['check']._consecutive_failures)
+ self.assertFalse(self.sched.layout.pipelines['check']._disabled)
+
+ # J and K went back to gerrit
+ self.assertEqual(1, len(J.messages))
+ self.assertIn('Build failed.', J.messages[0])
+ self.assertEqual(1, len(K.messages))
+ self.assertIn('Build failed.', K.messages[0])
+ # No more messages reported via smtp
+ self.assertEqual(3, len(self.smtp_messages))
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index 6e14ff5..59ac419 100644
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -263,6 +263,12 @@
'number': {
'title': 'Number'
},
+ 'node_labels': {
+ 'title': 'Node Labels'
+ },
+ 'node_name': {
+ 'title': 'Node Name'
+ },
'worker.name': {
'title': 'Worker'
},
@@ -276,7 +282,7 @@
'worker.fqdn': {
'title': 'Worker Domain'
},
- 'worker.progam': {
+ 'worker.program': {
'title': 'Worker Program'
},
'worker.version': {
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
index e4a0e7b..30401d6 100755
--- a/zuul/cmd/cloner.py
+++ b/zuul/cmd/cloner.py
@@ -51,8 +51,11 @@
version=self._get_version(),
help='show zuul version')
parser.add_argument('--cache-dir', dest='cache_dir',
+ default=os.environ.get('ZUUL_CACHE_DIR'),
help=('a directory that holds cached copies of '
- 'repos from which to make an initial clone.'
+ 'repos from which to make an initial clone. '
+ 'Can also be set via ZUUL_CACHE_DIR '
+ 'environment variable.'
))
parser.add_argument('git_base_url',
help='reference repo to clone from')
diff --git a/zuul/launcher/gearman.py b/zuul/launcher/gearman.py
index 828e2a9..69fb71b 100644
--- a/zuul/launcher/gearman.py
+++ b/zuul/launcher/gearman.py
@@ -277,6 +277,7 @@
ZUUL_PROJECT=item.change.project.name)
params['ZUUL_PIPELINE'] = pipeline.name
params['ZUUL_URL'] = item.current_build_set.zuul_url
+ params['ZUUL_VOTING'] = job.voting and '1' or '0'
if hasattr(item.change, 'refspec'):
changes_str = '^'.join(
['%s:%s:%s' % (i.change.project.name, i.change.branch,
@@ -422,9 +423,11 @@
build = self.builds.get(job.unique)
if build:
+ data = getJobData(job)
+ build.node_labels = data.get('node_labels', [])
+ build.node_name = data.get('node_name')
if not build.canceled:
if result is None:
- data = getJobData(job)
result = data.get('result')
if result is None:
build.retry = True
diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py
index 88d10e2..781f475 100644
--- a/zuul/layoutvalidator.py
+++ b/zuul/layoutvalidator.py
@@ -61,6 +61,7 @@
'username': toList(str),
'branch': toList(str),
'ref': toList(str),
+ 'ignore-deletes': bool,
'approval': toList(variable_dict),
'require-approval': toList(require_approval),
}
@@ -112,6 +113,9 @@
'failure': report_actions,
'merge-failure': report_actions,
'start': report_actions,
+ 'disabled': report_actions,
+ 'disable-after-consecutive-failures':
+ v.All(int, v.Range(min=1)),
'window': window,
'window-floor': window_floor,
'window-increase-type': window_type,
diff --git a/zuul/lib/gerrit.py b/zuul/lib/gerrit.py
index 5821a2d..1434389 100644
--- a/zuul/lib/gerrit.py
+++ b/zuul/lib/gerrit.py
@@ -204,7 +204,7 @@
key_filename=self.keyfile)
self.client = client
- def _ssh(self, command):
+ def _ssh(self, command, stdin_data=None):
if not self.client:
self._open()
@@ -215,6 +215,9 @@
self._open()
stdin, stdout, stderr = self.client.exec_command(command)
+ if stdin_data:
+ stdin.write(stdin_data)
+
out = stdout.read()
self.log.debug("SSH received stdout:\n%s" % out)
diff --git a/zuul/model.py b/zuul/model.py
index f8e0d25..1b94029 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -22,6 +22,8 @@
'ordereddict.OrderedDict'])
+EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
+
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
MERGER_CHERRY_PICK = 3 # "git cherry-pick"
@@ -82,6 +84,10 @@
self.start_actions = None
self.success_actions = None
self.failure_actions = None
+ self.disabled_actions = None
+ self.disable_at = None
+ self._consecutive_failures = 0
+ self._disabled = False
self.window = None
self.window_floor = None
self.window_increase_type = None
@@ -431,9 +437,13 @@
class Project(object):
- def __init__(self, name):
+ def __init__(self, name, foreign=False):
self.name = name
self.merge_mode = MERGER_MERGE_RESOLVE
+ # foreign projects are those referenced in dependencies
+ # of layout projects, this should matter
+ # when deciding whether to enqueue their changes
+ self.foreign = foreign
def __str__(self):
return self.name
@@ -581,6 +591,8 @@
self.retry = False
self.parameters = {}
self.worker = Worker()
+ self.node_labels = []
+ self.node_name = None
def __repr__(self):
return ('<Build %s of %s on %s>' %
@@ -802,7 +814,9 @@
'canceled': build.canceled if build else None,
'retry': build.retry if build else None,
'number': build.number if build else None,
- 'worker': worker
+ 'node_labels': build.node_labels if build else [],
+ 'node_name': build.node_name if build else None,
+ 'worker': worker,
})
if self.pipeline.haveAllJobsStarted(self):
@@ -1087,7 +1101,8 @@
class EventFilter(BaseFilter):
def __init__(self, trigger, types=[], branches=[], refs=[],
event_approvals={}, comments=[], emails=[], usernames=[],
- timespecs=[], required_approvals=[], pipelines=[]):
+ timespecs=[], required_approvals=[], pipelines=[],
+ ignore_deletes=True):
super(EventFilter, self).__init__(
required_approvals=required_approvals)
self.trigger = trigger
@@ -1107,6 +1122,7 @@
self.pipelines = [re.compile(x) for x in pipelines]
self.event_approvals = event_approvals
self.timespecs = timespecs
+ self.ignore_deletes = ignore_deletes
def __repr__(self):
ret = '<EventFilter'
@@ -1119,6 +1135,8 @@
ret += ' branches: %s' % ', '.join(self._branches)
if self._refs:
ret += ' refs: %s' % ', '.join(self._refs)
+ if self.ignore_deletes:
+ ret += ' ignore_deletes: %s' % self.ignore_deletes
if self.event_approvals:
ret += ' event_approvals: %s' % ', '.join(
['%s:%s' % a for a in self.event_approvals.items()])
@@ -1170,6 +1188,10 @@
matches_ref = True
if self.refs and not matches_ref:
return False
+ if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
+ # If the updated ref has an empty git sha (all 0s),
+ # then the ref is being deleted
+ return False
# comments are ORed
matches_comment_re = False
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a9bd6b2..6987e7f 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -286,7 +286,8 @@
'ignore-dependencies', False)
action_reporters = {}
- for action in ['start', 'success', 'failure', 'merge-failure']:
+ for action in ['start', 'success', 'failure', 'merge-failure',
+ 'disabled']:
action_reporters[action] = []
if conf_pipeline.get(action):
for reporter_name, params \
@@ -300,12 +301,16 @@
pipeline.start_actions = action_reporters['start']
pipeline.success_actions = action_reporters['success']
pipeline.failure_actions = action_reporters['failure']
+ pipeline.disabled_actions = action_reporters['disabled']
if len(action_reporters['merge-failure']) > 0:
pipeline.merge_failure_actions = \
action_reporters['merge-failure']
else:
pipeline.merge_failure_actions = action_reporters['failure']
+ pipeline.disable_at = conf_pipeline.get(
+ 'disable-after-consecutive-failures', None)
+
pipeline.window = conf_pipeline.get('window', 20)
pipeline.window_floor = conf_pipeline.get('window-floor', 3)
pipeline.window_increase_type = conf_pipeline.get(
@@ -348,6 +353,7 @@
usernames = toList(trigger.get('username'))
if not usernames:
usernames = toList(trigger.get('username_filter'))
+ ignore_deletes = trigger.get('ignore-deletes', True)
f = EventFilter(
trigger=self.triggers['gerrit'],
types=toList(trigger['event']),
@@ -359,7 +365,8 @@
usernames=usernames,
required_approvals=toList(
trigger.get('require-approval')
- )
+ ),
+ ignore_deletes=ignore_deletes
)
manager.event_filters.append(f)
if 'timer' in conf_pipeline['trigger']:
@@ -507,11 +514,15 @@
name = reporter.name
self.reporters[name] = reporter
- def getProject(self, name):
+ def getProject(self, name, create_foreign=False):
self.layout_lock.acquire()
p = None
try:
p = self.layout.projects.get(name)
+ if p is None and create_foreign:
+ self.log.info("Registering foreign project: %s" % name)
+ p = Project(name, foreign=True)
+ self.layout.projects[name] = p
finally:
self.layout_lock.release()
return p
@@ -530,6 +541,16 @@
def onBuildStarted(self, build):
self.log.debug("Adding start event for build: %s" % build)
build.start_time = time.time()
+ try:
+ if statsd and build.pipeline:
+ jobname = build.job.name.replace('.', '_')
+ key = 'zuul.pipeline.%s.job.%s.wait_time' % (
+ build.pipeline.name, jobname)
+ dt = int((build.start_time - build.launch_time) * 1000)
+ statsd.timing(key, dt)
+ statsd.incr(key)
+ except:
+ self.log.exception("Exception reporting runtime stats")
event = BuildStartedEvent(build)
self.result_event_queue.put(event)
self.wake_event.set()
@@ -547,14 +568,25 @@
try:
if statsd and build.pipeline:
jobname = build.job.name.replace('.', '_')
+ key = 'zuul.pipeline.%s.all_jobs' % build.pipeline.name
+ statsd.incr(key)
+ for label in build.node_labels:
+ # Jenkins includes the node name in its list of labels, so
+ # we filter it out here, since that is not statistically
+ # interesting.
+ if label == build.node_name:
+ continue
+ dt = int((build.start_time - build.launch_time) * 1000)
+ key = 'zuul.node_type.%s.job.%s.wait_time' % (
+ label, jobname)
+ statsd.timing(key, dt)
+ statsd.incr(key)
key = 'zuul.pipeline.%s.job.%s.%s' % (build.pipeline.name,
jobname, build.result)
if build.result in ['SUCCESS', 'FAILURE'] and build.start_time:
dt = int((build.end_time - build.start_time) * 1000)
statsd.timing(key, dt)
statsd.incr(key)
- key = 'zuul.pipeline.%s.all_jobs' % build.pipeline.name
- statsd.incr(key)
except:
self.log.exception("Exception reporting runtime stats")
event = BuildCompletedEvent(build)
@@ -685,15 +717,15 @@
item.items_behind = []
item.pipeline = None
item.queue = None
- project = layout.projects.get(item.change.project.name)
- if not project:
- self.log.warning("Unable to find project for "
- "change %s while reenqueueing" %
- item.change)
- item.change.project = None
- items_to_remove.append(item)
- continue
- item.change.project = project
+ project_name = item.change.project.name
+ item.change.project = layout.projects.get(project_name)
+ if not item.change.project:
+ self.log.debug("Project %s not defined, "
+ "re-instantiating as foreign" %
+ project_name)
+ project = Project(project_name, foreign=True)
+ layout.projects[project_name] = project
+ item.change.project = project
item_jobs = new_pipeline.getJobs(item)
for build in item.current_build_set.getBuilds():
job = layout.jobs.get(build.job.name)
@@ -861,7 +893,7 @@
self.log.debug("Processing trigger event %s" % event)
try:
project = self.layout.projects.get(event.project_name)
- if not project:
+ if not project or project.foreign:
self.log.debug("Project %s not found" % event.project_name)
return
@@ -1045,6 +1077,8 @@
self.log.info(" %s" % self.pipeline.failure_actions)
self.log.info(" On merge-failure:")
self.log.info(" %s" % self.pipeline.merge_failure_actions)
+ self.log.info(" When disabled:")
+ self.log.info(" %s" % self.pipeline.disabled_actions)
def getSubmitAllowNeeds(self):
# Get a list of code review labels that are allowed to be
@@ -1086,19 +1120,20 @@
return False
def reportStart(self, change):
- try:
- self.log.info("Reporting start, action %s change %s" %
- (self.pipeline.start_actions, change))
- msg = "Starting %s jobs." % self.pipeline.name
- if self.sched.config.has_option('zuul', 'status_url'):
- msg += "\n" + self.sched.config.get('zuul', 'status_url')
- ret = self.sendReport(self.pipeline.start_actions,
- change, msg)
- if ret:
- self.log.error("Reporting change start %s received: %s" %
- (change, ret))
- except:
- self.log.exception("Exception while reporting start:")
+ if not self.pipeline._disabled:
+ try:
+ self.log.info("Reporting start, action %s change %s" %
+ (self.pipeline.start_actions, change))
+ msg = "Starting %s jobs." % self.pipeline.name
+ if self.sched.config.has_option('zuul', 'status_url'):
+ msg += "\n" + self.sched.config.get('zuul', 'status_url')
+ ret = self.sendReport(self.pipeline.start_actions,
+ change, msg)
+ if ret:
+ self.log.error("Reporting change start %s received: %s" %
+ (change, ret))
+ except:
+ self.log.exception("Exception while reporting start:")
def sendReport(self, action_reporters, change, message):
"""Sends the built message off to configured reporters.
@@ -1544,12 +1579,22 @@
self.log.debug("success %s" % (self.pipeline.success_actions))
actions = self.pipeline.success_actions
item.setReportedResult('SUCCESS')
+ self.pipeline._consecutive_failures = 0
elif not self.pipeline.didMergerSucceed(item):
actions = self.pipeline.merge_failure_actions
item.setReportedResult('MERGER_FAILURE')
else:
actions = self.pipeline.failure_actions
item.setReportedResult('FAILURE')
+ self.pipeline._consecutive_failures += 1
+ if self.pipeline._disabled:
+ actions = self.pipeline.disabled_actions
+ # Check here if we should disable so that we only use the disabled
+ # reporters /after/ the last disable_at failure is still reported as
+ # normal.
+ if (self.pipeline.disable_at and not self.pipeline._disabled and
+ self.pipeline._consecutive_failures >= self.pipeline.disable_at):
+ self.pipeline._disabled = True
if actions:
report = self.formatReport(item)
try:
@@ -1797,10 +1842,11 @@
if existing:
return DynamicChangeQueueContextManager(existing)
if change.project not in self.pipeline.getProjects():
- return DynamicChangeQueueContextManager(None)
+ self.pipeline.addProject(change.project)
change_queue = ChangeQueue(self.pipeline)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
+ self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def enqueueChangesAhead(self, change, quiet, ignore_requirements,
diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py
index 175e3f8..05d7581 100644
--- a/zuul/trigger/gerrit.py
+++ b/zuul/trigger/gerrit.py
@@ -94,7 +94,7 @@
Can not get account information." % event.type)
event.account = None
- if event.change_number:
+ if event.change_number and self.sched.getProject(event.project_name):
# Call _getChange for the side effect of updating the
# cache. Note that this modifies Change objects outside
# the main thread.
@@ -404,7 +404,11 @@
if 'project' not in data:
raise Exception("Change %s,%s not found" % (change.number,
change.patchset))
- change.project = self.sched.getProject(data['project'])
+ # If updated changed came as a dependent on
+ # and its project is not defined,
+ # then create a 'foreign' project for it in layout
+ change.project = self.sched.getProject(data['project'],
+ create_foreign=bool(history))
change.branch = data['branch']
change.url = data['url']
max_ps = 0