Merge "Really change patchset column to string" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 3bec28a..18bbfa3 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -408,7 +408,7 @@
Path to command socket file for the executor process.
.. attr:: finger_port
- :default: 79
+ :default: 7900
Port to use for finger log streamer.
@@ -451,13 +451,6 @@
SSH private key file to be used when logging into worker nodes.
- .. attr:: user
- :default: zuul
-
- User ID for the zuul-executor process. In normal operation as a
- daemon, the executor should be started as the ``root`` user, but
- it will drop privileges to this user during startup.
-
.. _admin_sitewide_variables:
.. attr:: variables
diff --git a/doc/source/admin/drivers/zuul.rst b/doc/source/admin/drivers/zuul.rst
index d95dffc..41535ee 100644
--- a/doc/source/admin/drivers/zuul.rst
+++ b/doc/source/admin/drivers/zuul.rst
@@ -26,6 +26,12 @@
When Zuul merges a change to a project, it generates this
event for every open change in the project.
+ .. warning::
+
+ Triggering on this event can cause poor performance when
+ using the GitHub driver with a large number of
+ installations.
+
.. value:: parent-change-enqueued
When Zuul enqueues a change into any pipeline, it generates
diff --git a/requirements.txt b/requirements.txt
index 39a2b02..193c64e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,5 +25,6 @@
cachecontrol
pyjwt
iso8601
+yarl>=0.11,<1.0
aiohttp
uvloop;python_version>='3.5'
diff --git a/tests/base.py b/tests/base.py
index 59c0d2a..c449242 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -170,7 +170,7 @@
'status': status,
'subject': subject,
'submitRecords': [],
- 'url': 'https://hostname/%s' % number}
+ 'url': 'https://%s/%s' % (self.gerrit.server, number)}
self.upstream_root = upstream_root
self.addPatchset(files=files, parent=parent)
@@ -559,14 +559,13 @@
return change.query()
return {}
- def simpleQuery(self, query):
- self.log.debug("simpleQuery: %s" % query)
- self.queries.append(query)
+ def _simpleQuery(self, query):
if query.startswith('change:'):
# Query a specific changeid
changeid = query[len('change:'):]
l = [change.query() for change in self.changes.values()
- if change.data['id'] == changeid]
+ if (change.data['id'] == changeid or
+ change.data['number'] == changeid)]
elif query.startswith('message:'):
# Query the content of a commit message
msg = query[len('message:'):].strip()
@@ -577,6 +576,20 @@
l = [change.query() for change in self.changes.values()]
return l
+ def simpleQuery(self, query):
+ self.log.debug("simpleQuery: %s" % query)
+ self.queries.append(query)
+ results = []
+ if query.startswith('(') and 'OR' in query:
+ query = query[1:-2]
+ for q in query.split(' OR '):
+ for r in self._simpleQuery(q):
+ if r not in results:
+ results.append(r)
+ else:
+ results = self._simpleQuery(query)
+ return results
+
def _start_watcher_thread(self, *args, **kw):
pass
@@ -628,6 +641,7 @@
self.is_merged = False
self.merge_message = None
self.state = 'open'
+ self.url = 'https://%s/%s/pull/%s' % (github.server, project, number)
self._createPRRef()
self._addCommitToRepo(files=files)
self._updateTimeStamp()
@@ -1605,6 +1619,8 @@
data['username'] = 'fakeuser'
if 'windows' in node_type:
data['connection_type'] = 'winrm'
+ if 'network' in node_type:
+ data['connection_type'] = 'network_cli'
data = json.dumps(data).encode('utf8')
path = self.client.create(path, data,
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-merge.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-test1.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-test2.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/nonvoting-project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/project-merge.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/project-post.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-post.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-post.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/project-testfile.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-testfile.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/project-testfile.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/playbooks/project1-project2-integration.yaml b/tests/fixtures/config/cross-source/git/common-config/playbooks/project1-project2-integration.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/playbooks/project1-project2-integration.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/cross-source/git/common-config/zuul.yaml b/tests/fixtures/config/cross-source/git/common-config/zuul.yaml
new file mode 100644
index 0000000..abdc34a
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/common-config/zuul.yaml
@@ -0,0 +1,168 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ github:
+ - event: pull_request
+ action: edited
+ success:
+ gerrit:
+ Verified: 1
+ github: {}
+ failure:
+ gerrit:
+ Verified: -1
+ github: {}
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ require:
+ github:
+ label: approved
+ gerrit:
+ approval:
+ - Approved: 1
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ github:
+ - event: pull_request
+ action: edited
+ - event: pull_request
+ action: labeled
+ label: approved
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ github:
+ merge: true
+ failure:
+ gerrit:
+ Verified: -2
+ github: {}
+ start:
+ gerrit:
+ Verified: 0
+ github: {}
+ precedence: high
+
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ gerrit:
+ - event: ref-updated
+ ref: ^(?!refs/).*$
+ precedence: low
+
+- job:
+ name: base
+ parent: null
+
+- job:
+ name: project-merge
+ hold-following-changes: true
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
+ run: playbooks/project-merge.yaml
+
+- job:
+ name: project-test1
+ attempts: 4
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
+ run: playbooks/project-test1.yaml
+
+- job:
+ name: project-test1
+ branches: stable
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
+ run: playbooks/project-test1.yaml
+
+- job:
+ name: project-post
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
+ run: playbooks/project-post.yaml
+
+- job:
+ name: project-test2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
+ run: playbooks/project-test2.yaml
+
+- job:
+ name: project1-project2-integration
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
+ run: playbooks/project1-project2-integration.yaml
+
+- job:
+ name: project-testfile
+ files:
+ - .*-requires
+ run: playbooks/project-testfile.yaml
+
+- project:
+ name: gerrit/project1
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
+ gate:
+ queue: integrated
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
+
+- project:
+ name: github/project2
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
+ gate:
+ queue: integrated
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
diff --git a/tests/fixtures/config/cross-source/git/gerrit_project1/README b/tests/fixtures/config/cross-source/git/gerrit_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/gerrit_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/cross-source/git/github_project2/README b/tests/fixtures/config/cross-source/git/github_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/cross-source/git/github_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/cross-source/main.yaml b/tests/fixtures/config/cross-source/main.yaml
new file mode 100644
index 0000000..bf85c33
--- /dev/null
+++ b/tests/fixtures/config/cross-source/main.yaml
@@ -0,0 +1,11 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - gerrit/project1
+ github:
+ untrusted-projects:
+ - github/project2
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index 36789a3..f592eb4 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -40,6 +40,8 @@
label: fakeuser-label
- name: windows
label: windows-label
+ - name: network
+ label: network-label
- job:
name: base
diff --git a/tests/fixtures/zuul-gerrit-github.conf b/tests/fixtures/zuul-gerrit-github.conf
new file mode 100644
index 0000000..d3cbf7b
--- /dev/null
+++ b/tests/fixtures/zuul-gerrit-github.conf
@@ -0,0 +1,35 @@
+[gearman]
+server=127.0.0.1
+
+[statsd]
+# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
+# see: https://github.com/jsocol/pystatsd/issues/61
+server=127.0.0.1
+
+[scheduler]
+tenant_config=main.yaml
+
+[merger]
+git_dir=/tmp/zuul-test/merger-git
+git_user_email=zuul@example.com
+git_user_name=zuul
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=fake_id_rsa_path
+
+[connection github]
+driver=github
+webhook_token=0000000000000000000000000000000000000000
+
+[connection smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index 4a405fd..c45da94 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -119,7 +119,7 @@
self.assertEqual('SUCCESS', buildset0['result'])
self.assertEqual('Build succeeded.', buildset0['message'])
self.assertEqual('tenant-one', buildset0['tenant'])
- self.assertEqual('https://hostname/%d' % buildset0['change'],
+ self.assertEqual('https://review.example.com/%d' % buildset0['change'],
buildset0['ref_url'])
buildset0_builds = conn.execute(
diff --git a/tests/unit/test_cross_crd.py b/tests/unit/test_cross_crd.py
new file mode 100644
index 0000000..7d68989
--- /dev/null
+++ b/tests/unit/test_cross_crd.py
@@ -0,0 +1,950 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tests.base import (
+ ZuulTestCase,
+)
+
+
+class TestGerritToGithubCRD(ZuulTestCase):
+ config_file = 'zuul-gerrit-github.conf'
+ tenant_config_file = 'config/cross-source/main.yaml'
+
+ def test_crd_gate(self):
+ "Test cross-repo dependencies"
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'B')
+
+ A.addApproval('Code-Review', 2)
+
+ AM2 = self.fake_gerrit.addFakeChange('gerrit/project1', 'master',
+ 'AM2')
+ AM1 = self.fake_gerrit.addFakeChange('gerrit/project1', 'master',
+ 'AM1')
+ AM2.setMerged()
+ AM1.setMerged()
+
+ # A -> AM1 -> AM2
+ # A Depends-On: B
+ # M2 is here to make sure it is never queried. If it is, it
+ # means zuul is walking down the entire history of merged
+ # changes.
+
+ A.setDependsOn(AM1, 1)
+ AM1.setDependsOn(AM2, 1)
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+
+ for connection in self.connections.connections.values():
+ connection.maintainCache([])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addLabel('approved')
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(AM2.queried, 0)
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertTrue(B.is_merged)
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(len(B.comments), 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,%s 1,1' % B.head_sha)
+
+ def test_crd_branch(self):
+ "Test cross-repo dependencies in multiple branches"
+
+ self.create_branch('github/project2', 'mp')
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'B')
+ C1 = self.fake_github.openFakePullRequest('github/project2', 'mp',
+ 'C1')
+
+ A.addApproval('Code-Review', 2)
+
+ # A Depends-On: B+C1
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
+ A.subject, B.url, C1.url)
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addLabel('approved')
+ C1.addLabel('approved')
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertTrue(B.is_merged)
+ self.assertTrue(C1.is_merged)
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(len(B.comments), 2)
+ self.assertEqual(len(C1.comments), 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,%s 2,%s 1,1' %
+ (B.head_sha, C1.head_sha))
+
+ def test_crd_gate_reverse(self):
+ "Test reverse cross-repo dependencies"
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'B')
+ A.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+
+ self.executor_server.hold_jobs_in_build = True
+ A.addApproval('Approved', 1)
+ self.fake_github.emitEvent(B.addLabel('approved'))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertTrue(B.is_merged)
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(len(B.comments), 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,%s 1,1' %
+ (B.head_sha,))
+
+ def test_crd_cycle(self):
+ "Test cross-repo dependency cycles"
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ msg = "Depends-On: %s" % (A.data['url'],)
+ B = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'B', body=msg)
+ A.addApproval('Code-Review', 2)
+ B.addLabel('approved')
+
+ # A -> B -> A (via commit-depends)
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(len(B.comments), 0)
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+
+ def test_crd_gate_unknown(self):
+ "Test unknown projects in dependent pipeline"
+ self.init_repo("github/unknown", tag='init')
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest('github/unknown', 'master',
+ 'B')
+ A.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ event = B.addLabel('approved')
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ # Unknown projects cannot share a queue with any other
+ # since they don't have common jobs with any other (they have no jobs).
+ # Changes which depend on unknown project changes
+ # should not be processed in dependent pipeline
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(len(B.comments), 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Simulate change B being gated outside this layout Set the
+ # change merged before submitting the event so that when the
+ # event triggers a gerrit query to update the change, we get
+ # the information that it was merged.
+ B.setMerged('merged')
+ self.fake_github.emitEvent(event)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertTrue(B.is_merged)
+ self.assertEqual(len(B.comments), 0)
+
+ def test_crd_check(self):
+ "Test cross-repo dependencies in independent pipelines"
+ self.executor_server.hold_jobs_in_build = True
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest(
+ 'github/project2', 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.assertTrue(self.builds[0].hasChanges(A, B))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(len(B.comments), 0)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,%s 1,1' %
+ (B.head_sha,))
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_duplicate(self):
+ "Test duplicate check in independent pipelines"
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest(
+ 'github/project2', 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ check_pipeline = tenant.layout.pipelines['check']
+
+ # Add two dependent changes...
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...make sure the live one is not duplicated...
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...but the non-live one is able to be.
+ self.fake_github.emitEvent(B.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 3)
+
+ # Release jobs in order to avoid races with change A jobs
+ # finishing before change B jobs.
+ self.orderedRelease()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(len(B.comments), 1)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,%s 1,1' %
+ (B.head_sha,))
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,%s' %
+ (B.head_sha,))
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ self.assertIn('Build succeeded', A.messages[0])
+
+ def _test_crd_check_reconfiguration(self, project1, project2):
+ "Test cross-repo dependencies re-enqueued in independent pipelines"
+
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest(
+ 'github/project2', 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.sched.reconfigure(self.config)
+
+ # Make sure the items still share a change queue, and the
+ # first one is not live.
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
+ queue = tenant.layout.pipelines['check'].queues[0]
+ first_item = queue.queue[0]
+ for item in queue.queue:
+ self.assertEqual(item.queue, first_item.queue)
+ self.assertFalse(first_item.live)
+ self.assertTrue(queue.queue[1].live)
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertFalse(B.is_merged)
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(len(B.comments), 0)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,%s 1,1' %
+ (B.head_sha,))
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_reconfiguration(self):
+ self._test_crd_check_reconfiguration('org/project1', 'org/project2')
+
+ def test_crd_undefined_project(self):
+ """Test that undefined projects in dependencies are handled for
+ independent pipelines"""
+ # It's a hack for fake github,
+ # as it implies repo creation upon the creation of any change
+ self.init_repo("github/unknown", tag='init')
+ self._test_crd_check_reconfiguration('gerrit/project1',
+ 'github/unknown')
+
+ def test_crd_check_transitive(self):
+ "Test transitive cross-repo dependencies"
+ # Specifically, if A -> B -> C, and C gets a new patchset and
+ # A gets a new patchset, ensure the test of A,2 includes B,1
+ # and C,2 (not C,1 which would indicate stale data in the
+ # cache for B).
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ C = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'C')
+ # B Depends-On: C
+ msg = "Depends-On: %s" % (C.data['url'],)
+ B = self.fake_github.openFakePullRequest(
+ 'github/project2', 'master', 'B', body=msg)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,1 1,%s 1,1' %
+ (B.head_sha,))
+
+ self.fake_github.emitEvent(B.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,1 1,%s' %
+ (B.head_sha,))
+
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,1')
+
+ C.addPatchset()
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,2')
+
+ A.addPatchset()
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,2 1,%s 1,2' %
+ (B.head_sha,))
+
+ def test_crd_check_unknown(self):
+ "Test unknown projects in independent pipeline"
+ self.init_repo("github/unknown", tag='init')
+ A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A')
+ B = self.fake_github.openFakePullRequest(
+ 'github/unknown', 'master', 'B')
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+
+ # Make sure zuul has seen an event on B.
+ self.fake_github.emitEvent(B.getPullRequestEditedEvent())
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertFalse(B.is_merged)
+ self.assertEqual(len(B.comments), 0)
+
+ def test_crd_cycle_join(self):
+ "Test an updated change creates a cycle"
+ A = self.fake_github.openFakePullRequest(
+ 'github/project2', 'master', 'A')
+
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(len(A.comments), 1)
+
+ # Create B->A
+ B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.url)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Dep is there so zuul should have reported on B
+ self.assertEqual(B.reported, 1)
+
+ # Update A to add A->B (a cycle).
+ A.editBody('Depends-On: %s\n' % (B.data['url']))
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+
+ # Dependency cycle injected so zuul should not have reported again on A
+ self.assertEqual(len(A.comments), 1)
+
+ # Now if we update B to remove the depends-on, everything
+ # should be okay. B; A->B
+
+ B.addPatchset()
+ B.data['commitMessage'] = '%s\n' % (B.subject,)
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+
+ # Cycle was removed so now zuul should have reported again on A
+ self.assertEqual(len(A.comments), 2)
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 2)
+
+
+class TestGithubToGerritCRD(ZuulTestCase):
+ config_file = 'zuul-gerrit-github.conf'
+ tenant_config_file = 'config/cross-source/main.yaml'
+
+ def test_crd_gate(self):
+ "Test cross-repo dependencies"
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
+
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url']))
+
+ event = A.addLabel('approved')
+ self.fake_github.emitEvent(event)
+ self.waitUntilSettled()
+
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+
+ for connection in self.connections.connections.values():
+ connection.maintainCache([])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ self.fake_github.emitEvent(event)
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertTrue(A.is_merged)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(len(A.comments), 2)
+ self.assertEqual(B.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,1 1,%s' % A.head_sha)
+
+ def test_crd_branch(self):
+ "Test cross-repo dependencies in multiple branches"
+
+ self.create_branch('gerrit/project1', 'mp')
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
+ C1 = self.fake_gerrit.addFakeChange('gerrit/project1', 'mp', 'C1')
+
+ B.addApproval('Code-Review', 2)
+ C1.addApproval('Code-Review', 2)
+
+ # A Depends-On: B+C1
+ A.editBody('Depends-On: %s\nDepends-On: %s\n' % (
+ B.data['url'], C1.data['url']))
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ C1.addApproval('Approved', 1)
+ self.fake_github.emitEvent(A.addLabel('approved'))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertTrue(A.is_merged)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(C1.data['status'], 'MERGED')
+ self.assertEqual(len(A.comments), 2)
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(C1.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,1 2,1 1,%s' %
+ (A.head_sha,))
+
+ def test_crd_gate_reverse(self):
+ "Test reverse cross-repo dependencies"
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ self.fake_github.emitEvent(A.addLabel('approved'))
+ self.waitUntilSettled()
+
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+
+ self.executor_server.hold_jobs_in_build = True
+ A.addLabel('approved')
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertTrue(A.is_merged)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(len(A.comments), 2)
+ self.assertEqual(B.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,1 1,%s' %
+ (A.head_sha,))
+
+ def test_crd_cycle(self):
+ "Test cross-repo dependency cycles"
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.url)
+
+ B.addApproval('Code-Review', 2)
+ B.addApproval('Approved', 1)
+
+ # A -> B -> A (via commit-depends)
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ self.fake_github.emitEvent(A.addLabel('approved'))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(A.comments), 0)
+ self.assertEqual(B.reported, 0)
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+
+ def test_crd_gate_unknown(self):
+ "Test unknown projects in dependent pipeline"
+ self.init_repo("gerrit/unknown", tag='init')
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange('gerrit/unknown', 'master', 'B')
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ B.addApproval('Approved', 1)
+ event = A.addLabel('approved')
+ self.fake_github.emitEvent(event)
+ self.waitUntilSettled()
+
+ # Unknown projects cannot share a queue with any other
+ # since they don't have common jobs with any other (they have no jobs).
+ # Changes which depend on unknown project changes
+ # should not be processed in dependent pipeline
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(len(A.comments), 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Simulate change B being gated outside this layout Set the
+ # change merged before submitting the event so that when the
+ # event triggers a gerrit query to update the change, we get
+ # the information that it was merged.
+ B.setMerged()
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_github.emitEvent(event)
+ self.waitUntilSettled()
+
+ self.assertTrue(A.is_merged)
+ self.assertEqual(len(A.comments), 2)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 0)
+
+ def test_crd_check(self):
+ "Test cross-repo dependencies in independent pipelines"
+ self.executor_server.hold_jobs_in_build = True
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange(
+ 'gerrit/project1', 'master', 'B')
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.assertTrue(self.builds[0].hasChanges(A, B))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(len(A.comments), 1)
+ self.assertEqual(B.reported, 0)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,1 1,%s' %
+ (A.head_sha,))
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_duplicate(self):
+ "Test duplicate check in independent pipelines"
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange(
+ 'gerrit/project1', 'master', 'B')
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ check_pipeline = tenant.layout.pipelines['check']
+
+ # Add two dependent changes...
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...make sure the live one is not duplicated...
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...but the non-live one is able to be.
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 3)
+
+ # Release jobs in order to avoid races with change A jobs
+ # finishing before change B jobs.
+ self.orderedRelease()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(len(A.comments), 1)
+ self.assertEqual(B.reported, 1)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,1 1,%s' %
+ (A.head_sha,))
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'gerrit/project1').changes
+ self.assertEqual(changes, '1,1')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ self.assertIn('Build succeeded', A.comments[0])
+
+ def _test_crd_check_reconfiguration(self, project1, project2):
+ "Test cross-repo dependencies re-enqueued in independent pipelines"
+
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange(
+ 'gerrit/project1', 'master', 'B')
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+
+ self.sched.reconfigure(self.config)
+
+ # Make sure the items still share a change queue, and the
+ # first one is not live.
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
+ queue = tenant.layout.pipelines['check'].queues[0]
+ first_item = queue.queue[0]
+ for item in queue.queue:
+ self.assertEqual(item.queue, first_item.queue)
+ self.assertFalse(first_item.live)
+ self.assertTrue(queue.queue[1].live)
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertFalse(A.is_merged)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(len(A.comments), 1)
+ self.assertEqual(B.reported, 0)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'github/project2').changes
+ self.assertEqual(changes, '1,1 1,%s' %
+ (A.head_sha,))
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_reconfiguration(self):
+ self._test_crd_check_reconfiguration('org/project1', 'org/project2')
+
+ def test_crd_undefined_project(self):
+ """Test that undefined projects in dependencies are handled for
+ independent pipelines"""
+ # It's a hack for fake gerrit,
+ # as it implies repo creation upon the creation of any change
+ self.init_repo("gerrit/unknown", tag='init')
+ self._test_crd_check_reconfiguration('github/project2',
+ 'gerrit/unknown')
+
+ def test_crd_check_transitive(self):
+ "Test transitive cross-repo dependencies"
+ # Specifically, if A -> B -> C, and C gets a new patchset and
+ # A gets a new patchset, ensure the test of A,2 includes B,1
+ # and C,2 (not C,1 which would indicate stale data in the
+ # cache for B).
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B')
+ C = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'C')
+
+ # B Depends-On: C
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, C.url)
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,%s 1,1 1,%s' %
+ (C.head_sha, A.head_sha))
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,%s 1,1' %
+ (C.head_sha,))
+
+ self.fake_github.emitEvent(C.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,%s' %
+ (C.head_sha,))
+
+ new_c_head = C.head_sha
+ C.addCommit()
+ old_c_head = C.head_sha
+ self.assertNotEqual(old_c_head, new_c_head)
+ self.fake_github.emitEvent(C.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,%s' %
+ (C.head_sha,))
+
+ new_a_head = A.head_sha
+ A.addCommit()
+ old_a_head = A.head_sha
+ self.assertNotEqual(old_a_head, new_a_head)
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '2,%s 1,1 1,%s' %
+ (C.head_sha, A.head_sha,))
+
+ def test_crd_check_unknown(self):
+ "Test unknown projects in independent pipeline"
+ self.init_repo("gerrit/unknown", tag='init')
+ A = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'A')
+ B = self.fake_gerrit.addFakeChange(
+ 'gerrit/unknown', 'master', 'B')
+
+ # A Depends-On: B
+ A.editBody('Depends-On: %s\n' % (B.data['url'],))
+
+ # Make sure zuul has seen an event on B.
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.fake_github.emitEvent(A.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+
+ self.assertFalse(A.is_merged)
+ self.assertEqual(len(A.comments), 1)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 0)
+
+ def test_crd_cycle_join(self):
+ "Test an updated change creates a cycle"
+ A = self.fake_gerrit.addFakeChange(
+ 'gerrit/project1', 'master', 'A')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1)
+
+ # Create B->A
+ B = self.fake_github.openFakePullRequest('github/project2', 'master',
+ 'B')
+ B.editBody('Depends-On: %s\n' % (A.data['url'],))
+ self.fake_github.emitEvent(B.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+
+ # Dep is there so zuul should have reported on B
+ self.assertEqual(len(B.comments), 1)
+
+ # Update A to add A->B (a cycle).
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.url)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Dependency cycle injected so zuul should not have reported again on A
+ self.assertEqual(A.reported, 1)
+
+ # Now if we update B to remove the depends-on, everything
+ # should be okay. B; A->B
+
+ B.addCommit()
+ B.editBody('')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Cycle was removed so now zuul should have reported again on A
+ self.assertEqual(A.reported, 2)
+
+ self.fake_github.emitEvent(B.getPullRequestEditedEvent())
+ self.waitUntilSettled()
+ self.assertEqual(len(B.comments), 2)
diff --git a/tests/unit/test_gerrit_crd.py b/tests/unit/test_gerrit_crd.py
new file mode 100644
index 0000000..732bc3d
--- /dev/null
+++ b/tests/unit/test_gerrit_crd.py
@@ -0,0 +1,626 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tests.base import (
+ ZuulTestCase,
+ simple_layout,
+)
+
+
+class TestGerritCRD(ZuulTestCase):
+ tenant_config_file = 'config/single-tenant/main.yaml'
+
+ def test_crd_gate(self):
+ "Test cross-repo dependencies"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ AM2 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM2')
+ AM1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM1')
+ AM2.setMerged()
+ AM1.setMerged()
+
+ BM2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM2')
+ BM1 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM1')
+ BM2.setMerged()
+ BM1.setMerged()
+
+ # A -> AM1 -> AM2
+ # B -> BM1 -> BM2
+ # A Depends-On: B
+ # M2 is here to make sure it is never queried. If it is, it
+ # means zuul is walking down the entire history of merged
+ # changes.
+
+ B.setDependsOn(BM1, 1)
+ BM1.setDependsOn(BM2, 1)
+
+ A.setDependsOn(AM1, 1)
+ AM1.setDependsOn(AM2, 1)
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+
+ for connection in self.connections.connections.values():
+ connection.maintainCache([])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(AM2.queried, 0)
+ self.assertEqual(BM2.queried, 0)
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 1,1')
+
+ def test_crd_branch(self):
+ "Test cross-repo dependencies in multiple branches"
+
+ self.create_branch('org/project2', 'mp')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C1 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C1')
+
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+ C1.addApproval('Code-Review', 2)
+
+ # A Depends-On: B+C1
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
+ A.subject, B.data['url'], C1.data['url'])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ C1.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(C1.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(C1.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 3,1 1,1')
+
+ def test_crd_multiline(self):
+ "Test multiple depends-on lines in commit"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+ C.addApproval('Code-Review', 2)
+
+ # A Depends-On: B+C
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
+ A.subject, B.data['url'], C.data['url'])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ C.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(C.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(C.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 3,1 1,1')
+
+ def test_crd_unshared_gate(self):
+ "Test cross-repo dependencies in unshared gate queues"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ # A and B do not share a queue, make sure that A is unable to
+ # enqueue B (and therefore, A is unable to be enqueued).
+ B.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Enqueue and merge B alone.
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 2)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+
+ def test_crd_gate_reverse(self):
+ "Test reverse cross-repo dependencies"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+
+ self.executor_server.hold_jobs_in_build = True
+ A.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 1,1')
+
+ def test_crd_cycle(self):
+ "Test cross-repo dependency cycles"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A -> B -> A (via commit-depends)
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['url'])
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+
+ def test_crd_gate_unknown(self):
+ "Test unknown projects in dependent pipeline"
+ self.init_repo("org/unknown", tag='init')
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ B.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ # Unknown projects cannot share a queue with any other
+ # since they don't have common jobs with any other (they have no jobs).
+ # Changes which depend on unknown project changes
+ # should not be processed in dependent pipeline
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Simulate change B being gated outside this layout Set the
+ # change merged before submitting the event so that when the
+ # event triggers a gerrit query to update the change, we get
+ # the information that it was merged.
+ B.setMerged()
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 0)
+
+ def test_crd_check(self):
+ "Test cross-repo dependencies in independent pipelines"
+
+ self.executor_server.hold_jobs_in_build = True
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.assertTrue(self.builds[0].hasChanges(A, B))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 0)
+
+ self.assertEqual(self.history[0].changes, '2,1 1,1')
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_git_depends(self):
+ "Test single-repo dependencies in independent pipelines"
+ self.gearman_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+
+ # Add two git-dependent changes and make sure they both report
+ # success.
+ B.setDependsOn(A, 1)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.orderedRelease()
+ self.gearman_server.hold_jobs_in_build = False
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 1)
+
+ self.assertEqual(self.history[0].changes, '1,1')
+ self.assertEqual(self.history[-1].changes, '1,1 2,1')
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ self.assertIn('Build succeeded', A.messages[0])
+ self.assertIn('Build succeeded', B.messages[0])
+
+ def test_crd_check_duplicate(self):
+ "Test duplicate check in independent pipelines"
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ check_pipeline = tenant.layout.pipelines['check']
+
+ # Add two git-dependent changes...
+ B.setDependsOn(A, 1)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...make sure the live one is not duplicated...
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...but the non-live one is able to be.
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 3)
+
+ # Release jobs in order to avoid races with change A jobs
+ # finishing before change B jobs.
+ self.orderedRelease()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 1)
+
+ self.assertEqual(self.history[0].changes, '1,1 2,1')
+ self.assertEqual(self.history[1].changes, '1,1')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ self.assertIn('Build succeeded', A.messages[0])
+ self.assertIn('Build succeeded', B.messages[0])
+
+ def _test_crd_check_reconfiguration(self, project1, project2):
+ "Test cross-repo dependencies re-enqueued in independent pipelines"
+
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
+ B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.sched.reconfigure(self.config)
+
+ # Make sure the items still share a change queue, and the
+ # first one is not live.
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
+ queue = tenant.layout.pipelines['check'].queues[0]
+ first_item = queue.queue[0]
+ for item in queue.queue:
+ self.assertEqual(item.queue, first_item.queue)
+ self.assertFalse(first_item.live)
+ self.assertTrue(queue.queue[1].live)
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 0)
+
+ self.assertEqual(self.history[0].changes, '2,1 1,1')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_reconfiguration(self):
+ self._test_crd_check_reconfiguration('org/project1', 'org/project2')
+
+ def test_crd_undefined_project(self):
+ """Test that undefined projects in dependencies are handled for
+ independent pipelines"""
+ # It's a hack for fake gerrit,
+ # as it implies repo creation upon the creation of any change
+ self.init_repo("org/unknown", tag='init')
+ self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
+
+ @simple_layout('layouts/ignore-dependencies.yaml')
+ def test_crd_check_ignore_dependencies(self):
+ "Test cross-repo dependencies can be ignored"
+
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+ # C git-depends on B
+ C.setDependsOn(B, 1)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Make sure none of the items share a change queue, and all
+ # are live.
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ check_pipeline = tenant.layout.pipelines['check']
+ self.assertEqual(len(check_pipeline.queues), 3)
+ self.assertEqual(len(check_pipeline.getAllItems()), 3)
+ for item in check_pipeline.getAllItems():
+ self.assertTrue(item.live)
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(C.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 1)
+ self.assertEqual(C.reported, 1)
+
+ # Each job should have tested exactly one change
+ for job in self.history:
+ self.assertEqual(len(job.changes.split()), 1)
+
+ @simple_layout('layouts/three-projects.yaml')
+ def test_crd_check_transitive(self):
+ "Test transitive cross-repo dependencies"
+ # Specifically, if A -> B -> C, and C gets a new patchset and
+ # A gets a new patchset, ensure the test of A,2 includes B,1
+ # and C,2 (not C,1 which would indicate stale data in the
+ # cache for B).
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ # B Depends-On: C
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, C.data['url'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,1 2,1 1,1')
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,1 2,1')
+
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,1')
+
+ C.addPatchset()
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,2')
+
+ A.addPatchset()
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
+
+ def test_crd_check_unknown(self):
+ "Test unknown projects in independent pipeline"
+ self.init_repo("org/unknown", tag='init')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'D')
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+
+ # Make sure zuul has seen an event on B.
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 0)
+
+ def test_crd_cycle_join(self):
+ "Test an updated change creates a cycle"
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1)
+
+ # Create B->A
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['url'])
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Dep is there so zuul should have reported on B
+ self.assertEqual(B.reported, 1)
+
+ # Update A to add A->B (a cycle).
+ A.addPatchset()
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['url'])
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+
+ # Dependency cycle injected so zuul should not have reported again on A
+ self.assertEqual(A.reported, 1)
+
+ # Now if we update B to remove the depends-on, everything
+ # should be okay. B; A->B
+
+ B.addPatchset()
+ B.data['commitMessage'] = '%s\n' % (B.subject,)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+
+ # Cycle was removed so now zuul should have reported again on A
+ self.assertEqual(A.reported, 2)
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 2)
diff --git a/tests/unit/test_gerrit_legacy_crd.py b/tests/unit/test_gerrit_legacy_crd.py
new file mode 100644
index 0000000..c711e4d
--- /dev/null
+++ b/tests/unit/test_gerrit_legacy_crd.py
@@ -0,0 +1,629 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tests.base import (
+ ZuulTestCase,
+ simple_layout,
+)
+
+
+class TestGerritLegacyCRD(ZuulTestCase):
+ tenant_config_file = 'config/single-tenant/main.yaml'
+
+ def test_crd_gate(self):
+ "Test cross-repo dependencies"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ AM2 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM2')
+ AM1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM1')
+ AM2.setMerged()
+ AM1.setMerged()
+
+ BM2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM2')
+ BM1 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM1')
+ BM2.setMerged()
+ BM1.setMerged()
+
+ # A -> AM1 -> AM2
+ # B -> BM1 -> BM2
+ # A Depends-On: B
+ # M2 is here to make sure it is never queried. If it is, it
+ # means zuul is walking down the entire history of merged
+ # changes.
+
+ B.setDependsOn(BM1, 1)
+ BM1.setDependsOn(BM2, 1)
+
+ A.setDependsOn(AM1, 1)
+ AM1.setDependsOn(AM2, 1)
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+
+ for connection in self.connections.connections.values():
+ connection.maintainCache([])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(AM2.queried, 0)
+ self.assertEqual(BM2.queried, 0)
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 1,1')
+
+ def test_crd_branch(self):
+ "Test cross-repo dependencies in multiple branches"
+
+ self.create_branch('org/project2', 'mp')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C1 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C1')
+ C2 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C2',
+ status='ABANDONED')
+ C1.data['id'] = B.data['id']
+ C2.data['id'] = B.data['id']
+
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+ C1.addApproval('Code-Review', 2)
+
+ # A Depends-On: B+C1
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ C1.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(C1.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(C1.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 3,1 1,1')
+
+ def test_crd_multiline(self):
+ "Test multiple depends-on lines in commit"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+ C.addApproval('Code-Review', 2)
+
+ # A Depends-On: B+C
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
+ A.subject, B.data['id'], C.data['id'])
+
+ self.executor_server.hold_jobs_in_build = True
+ B.addApproval('Approved', 1)
+ C.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(C.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+ self.assertEqual(C.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 3,1 1,1')
+
+ def test_crd_unshared_gate(self):
+ "Test cross-repo dependencies in unshared gate queues"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ # A and B do not share a queue, make sure that A is unable to
+ # enqueue B (and therefore, A is unable to be enqueued).
+ B.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Enqueue and merge B alone.
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 2)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+
+ def test_crd_gate_reverse(self):
+ "Test reverse cross-repo dependencies"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+
+ self.executor_server.hold_jobs_in_build = True
+ A.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.reported, 2)
+
+ changes = self.getJobFromHistory(
+ 'project-merge', 'org/project1').changes
+ self.assertEqual(changes, '2,1 1,1')
+
+ def test_crd_cycle(self):
+ "Test cross-repo dependency cycles"
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A -> B -> A (via commit-depends)
+
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+
+ def test_crd_gate_unknown(self):
+ "Test unknown projects in dependent pipeline"
+ self.init_repo("org/unknown", tag='init')
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
+ A.addApproval('Code-Review', 2)
+ B.addApproval('Code-Review', 2)
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ B.addApproval('Approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ # Unknown projects cannot share a queue with any other
+ # since they don't have common jobs with any other (they have no jobs).
+ # Changes which depend on unknown project changes
+ # should not be processed in dependent pipeline
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 0)
+ self.assertEqual(B.reported, 0)
+ self.assertEqual(len(self.history), 0)
+
+ # Simulate change B being gated outside this layout Set the
+ # change merged before submitting the event so that when the
+ # event triggers a gerrit query to update the change, we get
+ # the information that it was merged.
+ B.setMerged()
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 0)
+
+ # Now that B is merged, A should be able to be enqueued and
+ # merged.
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 0)
+
+ def test_crd_check(self):
+ "Test cross-repo dependencies in independent pipelines"
+
+ self.executor_server.hold_jobs_in_build = True
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.executor_server.release('.*-merge')
+ self.waitUntilSettled()
+
+ self.assertTrue(self.builds[0].hasChanges(A, B))
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 0)
+
+ self.assertEqual(self.history[0].changes, '2,1 1,1')
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_git_depends(self):
+ "Test single-repo dependencies in independent pipelines"
+ self.gearman_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+
+ # Add two git-dependent changes and make sure they both report
+ # success.
+ B.setDependsOn(A, 1)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.orderedRelease()
+ self.gearman_server.hold_jobs_in_build = False
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 1)
+
+ self.assertEqual(self.history[0].changes, '1,1')
+ self.assertEqual(self.history[-1].changes, '1,1 2,1')
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ self.assertIn('Build succeeded', A.messages[0])
+ self.assertIn('Build succeeded', B.messages[0])
+
+ def test_crd_check_duplicate(self):
+ "Test duplicate check in independent pipelines"
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ check_pipeline = tenant.layout.pipelines['check']
+
+ # Add two git-dependent changes...
+ B.setDependsOn(A, 1)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...make sure the live one is not duplicated...
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 2)
+
+ # ...but the non-live one is able to be.
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(check_pipeline.getAllItems()), 3)
+
+ # Release jobs in order to avoid races with change A jobs
+ # finishing before change B jobs.
+ self.orderedRelease()
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 1)
+
+ self.assertEqual(self.history[0].changes, '1,1 2,1')
+ self.assertEqual(self.history[1].changes, '1,1')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ self.assertIn('Build succeeded', A.messages[0])
+ self.assertIn('Build succeeded', B.messages[0])
+
+ def _test_crd_check_reconfiguration(self, project1, project2):
+ "Test cross-repo dependencies re-enqueued in independent pipelines"
+
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
+ B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.sched.reconfigure(self.config)
+
+ # Make sure the items still share a change queue, and the
+ # first one is not live.
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
+ queue = tenant.layout.pipelines['check'].queues[0]
+ first_item = queue.queue[0]
+ for item in queue.queue:
+ self.assertEqual(item.queue, first_item.queue)
+ self.assertFalse(first_item.live)
+ self.assertTrue(queue.queue[1].live)
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 0)
+
+ self.assertEqual(self.history[0].changes, '2,1 1,1')
+ self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
+
+ def test_crd_check_reconfiguration(self):
+ self._test_crd_check_reconfiguration('org/project1', 'org/project2')
+
+ def test_crd_undefined_project(self):
+ """Test that undefined projects in dependencies are handled for
+ independent pipelines"""
+ # It's a hack for fake gerrit,
+ # as it implies repo creation upon the creation of any change
+ self.init_repo("org/unknown", tag='init')
+ self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
+
+ @simple_layout('layouts/ignore-dependencies.yaml')
+ def test_crd_check_ignore_dependencies(self):
+ "Test cross-repo dependencies can be ignored"
+
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+ # C git-depends on B
+ C.setDependsOn(B, 1)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Make sure none of the items share a change queue, and all
+ # are live.
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ check_pipeline = tenant.layout.pipelines['check']
+ self.assertEqual(len(check_pipeline.queues), 3)
+ self.assertEqual(len(check_pipeline.getAllItems()), 3)
+ for item in check_pipeline.getAllItems():
+ self.assertTrue(item.live)
+
+ self.gearman_server.hold_jobs_in_queue = False
+ self.gearman_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(C.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.reported, 1)
+ self.assertEqual(C.reported, 1)
+
+ # Each job should have tested exactly one change
+ for job in self.history:
+ self.assertEqual(len(job.changes.split()), 1)
+
+ @simple_layout('layouts/three-projects.yaml')
+ def test_crd_check_transitive(self):
+ "Test transitive cross-repo dependencies"
+ # Specifically, if A -> B -> C, and C gets a new patchset and
+ # A gets a new patchset, ensure the test of A,2 includes B,1
+ # and C,2 (not C,1 which would indicate stale data in the
+ # cache for B).
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ # B Depends-On: C
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, C.data['id'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,1 2,1 1,1')
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,1 2,1')
+
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,1')
+
+ C.addPatchset()
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,2')
+
+ A.addPatchset()
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
+
+ def test_crd_check_unknown(self):
+ "Test unknown projects in independent pipeline"
+ self.init_repo("org/unknown", tag='init')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'D')
+ # A Depends-On: B
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+
+ # Make sure zuul has seen an event on B.
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 0)
+
+ def test_crd_cycle_join(self):
+ "Test an updated change creates a cycle"
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1)
+
+ # Create B->A
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # Dep is there so zuul should have reported on B
+ self.assertEqual(B.reported, 1)
+
+ # Update A to add A->B (a cycle).
+ A.addPatchset()
+ A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ A.subject, B.data['id'])
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+
+ # Dependency cycle injected so zuul should not have reported again on A
+ self.assertEqual(A.reported, 1)
+
+ # Now if we update B to remove the depends-on, everything
+ # should be okay. B; A->B
+
+ B.addPatchset()
+ B.data['commitMessage'] = '%s\n' % (B.subject,)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+
+ # Cycle was removed so now zuul should have reported again on A
+ self.assertEqual(A.reported, 2)
+
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 2)
diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py
index be50447..b7e35eb 100644
--- a/tests/unit/test_inventory.py
+++ b/tests/unit/test_inventory.py
@@ -37,6 +37,12 @@
inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
return yaml.safe_load(open(inv_path, 'r'))
+ def _get_setup_inventory(self, name):
+ build = self.getBuildByName(name)
+ setup_inv_path = os.path.join(build.jobdir.root, 'ansible',
+ 'setup-inventory.yaml')
+ return yaml.safe_load(open(setup_inv_path, 'r'))
+
def test_single_inventory(self):
inventory = self._get_build_inventory('single-inventory')
@@ -131,3 +137,23 @@
self.executor_server.release()
self.waitUntilSettled()
+
+ def test_setup_inventory(self):
+
+ setup_inventory = self._get_setup_inventory('hostvars-inventory')
+ inventory = self._get_build_inventory('hostvars-inventory')
+
+ self.assertIn('all', inventory)
+ self.assertIn('hosts', inventory['all'])
+
+ self.assertIn('default', setup_inventory['all']['hosts'])
+ self.assertIn('fakeuser', setup_inventory['all']['hosts'])
+ self.assertIn('windows', setup_inventory['all']['hosts'])
+ self.assertNotIn('network', setup_inventory['all']['hosts'])
+ self.assertIn('default', inventory['all']['hosts'])
+ self.assertIn('fakeuser', inventory['all']['hosts'])
+ self.assertIn('windows', inventory['all']['hosts'])
+ self.assertIn('network', inventory['all']['hosts'])
+
+ self.executor_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 6bbf098..5db20b3 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -4196,7 +4196,7 @@
running_item = running_items[0]
self.assertEqual([], running_item['failing_reasons'])
self.assertEqual([], running_item['items_behind'])
- self.assertEqual('https://hostname/1', running_item['url'])
+ self.assertEqual('https://review.example.com/1', running_item['url'])
self.assertIsNone(running_item['item_ahead'])
self.assertEqual('org/project', running_item['project'])
self.assertIsNone(running_item['remaining_time'])
@@ -4247,611 +4247,6 @@
'SUCCESS')
self.assertEqual(A.reported, 1)
- def test_crd_gate(self):
- "Test cross-repo dependencies"
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
-
- AM2 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM2')
- AM1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM1')
- AM2.setMerged()
- AM1.setMerged()
-
- BM2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM2')
- BM1 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM1')
- BM2.setMerged()
- BM1.setMerged()
-
- # A -> AM1 -> AM2
- # B -> BM1 -> BM2
- # A Depends-On: B
- # M2 is here to make sure it is never queried. If it is, it
- # means zuul is walking down the entire history of merged
- # changes.
-
- B.setDependsOn(BM1, 1)
- BM1.setDependsOn(BM2, 1)
-
- A.setDependsOn(AM1, 1)
- AM1.setDependsOn(AM2, 1)
-
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
-
- for connection in self.connections.connections.values():
- connection.maintainCache([])
-
- self.executor_server.hold_jobs_in_build = True
- B.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(AM2.queried, 0)
- self.assertEqual(BM2.queried, 0)
- self.assertEqual(A.data['status'], 'MERGED')
- self.assertEqual(B.data['status'], 'MERGED')
- self.assertEqual(A.reported, 2)
- self.assertEqual(B.reported, 2)
-
- changes = self.getJobFromHistory(
- 'project-merge', 'org/project1').changes
- self.assertEqual(changes, '2,1 1,1')
-
- def test_crd_branch(self):
- "Test cross-repo dependencies in multiple branches"
-
- self.create_branch('org/project2', 'mp')
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- C1 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C1')
- C2 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C2',
- status='ABANDONED')
- C1.data['id'] = B.data['id']
- C2.data['id'] = B.data['id']
-
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
- C1.addApproval('Code-Review', 2)
-
- # A Depends-On: B+C1
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- self.executor_server.hold_jobs_in_build = True
- B.addApproval('Approved', 1)
- C1.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'MERGED')
- self.assertEqual(B.data['status'], 'MERGED')
- self.assertEqual(C1.data['status'], 'MERGED')
- self.assertEqual(A.reported, 2)
- self.assertEqual(B.reported, 2)
- self.assertEqual(C1.reported, 2)
-
- changes = self.getJobFromHistory(
- 'project-merge', 'org/project1').changes
- self.assertEqual(changes, '2,1 3,1 1,1')
-
- def test_crd_multiline(self):
- "Test multiple depends-on lines in commit"
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
- C.addApproval('Code-Review', 2)
-
- # A Depends-On: B+C
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
- A.subject, B.data['id'], C.data['id'])
-
- self.executor_server.hold_jobs_in_build = True
- B.addApproval('Approved', 1)
- C.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'MERGED')
- self.assertEqual(B.data['status'], 'MERGED')
- self.assertEqual(C.data['status'], 'MERGED')
- self.assertEqual(A.reported, 2)
- self.assertEqual(B.reported, 2)
- self.assertEqual(C.reported, 2)
-
- changes = self.getJobFromHistory(
- 'project-merge', 'org/project1').changes
- self.assertEqual(changes, '2,1 3,1 1,1')
-
- def test_crd_unshared_gate(self):
- "Test cross-repo dependencies in unshared gate queues"
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
-
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- # A and B do not share a queue, make sure that A is unable to
- # enqueue B (and therefore, A is unable to be enqueued).
- B.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(A.reported, 0)
- self.assertEqual(B.reported, 0)
- self.assertEqual(len(self.history), 0)
-
- # Enqueue and merge B alone.
- self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(B.data['status'], 'MERGED')
- self.assertEqual(B.reported, 2)
-
- # Now that B is merged, A should be able to be enqueued and
- # merged.
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'MERGED')
- self.assertEqual(A.reported, 2)
-
- def test_crd_gate_reverse(self):
- "Test reverse cross-repo dependencies"
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
-
- # A Depends-On: B
-
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
-
- self.executor_server.hold_jobs_in_build = True
- A.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'MERGED')
- self.assertEqual(B.data['status'], 'MERGED')
- self.assertEqual(A.reported, 2)
- self.assertEqual(B.reported, 2)
-
- changes = self.getJobFromHistory(
- 'project-merge', 'org/project1').changes
- self.assertEqual(changes, '2,1 1,1')
-
- def test_crd_cycle(self):
- "Test cross-repo dependency cycles"
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
-
- # A -> B -> A (via commit-depends)
-
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
- B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- B.subject, A.data['id'])
-
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(A.reported, 0)
- self.assertEqual(B.reported, 0)
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
-
- def test_crd_gate_unknown(self):
- "Test unknown projects in dependent pipeline"
- self.init_repo("org/unknown", tag='init')
- A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
- A.addApproval('Code-Review', 2)
- B.addApproval('Code-Review', 2)
-
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- B.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- # Unknown projects cannot share a queue with any other
- # since they don't have common jobs with any other (they have no jobs).
- # Changes which depend on unknown project changes
- # should not be processed in dependent pipeline
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(A.reported, 0)
- self.assertEqual(B.reported, 0)
- self.assertEqual(len(self.history), 0)
-
- # Simulate change B being gated outside this layout Set the
- # change merged before submitting the event so that when the
- # event triggers a gerrit query to update the change, we get
- # the information that it was merged.
- B.setMerged()
- self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
- self.waitUntilSettled()
- self.assertEqual(len(self.history), 0)
-
- # Now that B is merged, A should be able to be enqueued and
- # merged.
- self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'MERGED')
- self.assertEqual(A.reported, 2)
- self.assertEqual(B.data['status'], 'MERGED')
- self.assertEqual(B.reported, 0)
-
- def test_crd_check(self):
- "Test cross-repo dependencies in independent pipelines"
-
- self.executor_server.hold_jobs_in_build = True
- self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
-
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- self.gearman_server.hold_jobs_in_queue = False
- self.gearman_server.release()
- self.waitUntilSettled()
-
- self.executor_server.release('.*-merge')
- self.waitUntilSettled()
-
- self.assertTrue(self.builds[0].hasChanges(A, B))
-
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(A.reported, 1)
- self.assertEqual(B.reported, 0)
-
- self.assertEqual(self.history[0].changes, '2,1 1,1')
- tenant = self.sched.abide.tenants.get('tenant-one')
- self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
-
- def test_crd_check_git_depends(self):
- "Test single-repo dependencies in independent pipelines"
- self.gearman_server.hold_jobs_in_build = True
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
-
- # Add two git-dependent changes and make sure they both report
- # success.
- B.setDependsOn(A, 1)
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- self.orderedRelease()
- self.gearman_server.hold_jobs_in_build = False
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(A.reported, 1)
- self.assertEqual(B.reported, 1)
-
- self.assertEqual(self.history[0].changes, '1,1')
- self.assertEqual(self.history[-1].changes, '1,1 2,1')
- tenant = self.sched.abide.tenants.get('tenant-one')
- self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
-
- self.assertIn('Build succeeded', A.messages[0])
- self.assertIn('Build succeeded', B.messages[0])
-
- def test_crd_check_duplicate(self):
- "Test duplicate check in independent pipelines"
- self.executor_server.hold_jobs_in_build = True
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
- tenant = self.sched.abide.tenants.get('tenant-one')
- check_pipeline = tenant.layout.pipelines['check']
-
- # Add two git-dependent changes...
- B.setDependsOn(A, 1)
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(len(check_pipeline.getAllItems()), 2)
-
- # ...make sure the live one is not duplicated...
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(len(check_pipeline.getAllItems()), 2)
-
- # ...but the non-live one is able to be.
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(len(check_pipeline.getAllItems()), 3)
-
- # Release jobs in order to avoid races with change A jobs
- # finishing before change B jobs.
- self.orderedRelease()
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(A.reported, 1)
- self.assertEqual(B.reported, 1)
-
- self.assertEqual(self.history[0].changes, '1,1 2,1')
- self.assertEqual(self.history[1].changes, '1,1')
- self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
-
- self.assertIn('Build succeeded', A.messages[0])
- self.assertIn('Build succeeded', B.messages[0])
-
- def _test_crd_check_reconfiguration(self, project1, project2):
- "Test cross-repo dependencies re-enqueued in independent pipelines"
-
- self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
- B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
-
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- self.sched.reconfigure(self.config)
-
- # Make sure the items still share a change queue, and the
- # first one is not live.
- tenant = self.sched.abide.tenants.get('tenant-one')
- self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
- queue = tenant.layout.pipelines['check'].queues[0]
- first_item = queue.queue[0]
- for item in queue.queue:
- self.assertEqual(item.queue, first_item.queue)
- self.assertFalse(first_item.live)
- self.assertTrue(queue.queue[1].live)
-
- self.gearman_server.hold_jobs_in_queue = False
- self.gearman_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(A.reported, 1)
- self.assertEqual(B.reported, 0)
-
- self.assertEqual(self.history[0].changes, '2,1 1,1')
- self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
-
- def test_crd_check_reconfiguration(self):
- self._test_crd_check_reconfiguration('org/project1', 'org/project2')
-
- def test_crd_undefined_project(self):
- """Test that undefined projects in dependencies are handled for
- independent pipelines"""
- # It's a hack for fake gerrit,
- # as it implies repo creation upon the creation of any change
- self.init_repo("org/unknown", tag='init')
- self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
-
- @simple_layout('layouts/ignore-dependencies.yaml')
- def test_crd_check_ignore_dependencies(self):
- "Test cross-repo dependencies can be ignored"
-
- self.gearman_server.hold_jobs_in_queue = True
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
-
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
- # C git-depends on B
- C.setDependsOn(B, 1)
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- # Make sure none of the items share a change queue, and all
- # are live.
- tenant = self.sched.abide.tenants.get('tenant-one')
- check_pipeline = tenant.layout.pipelines['check']
- self.assertEqual(len(check_pipeline.queues), 3)
- self.assertEqual(len(check_pipeline.getAllItems()), 3)
- for item in check_pipeline.getAllItems():
- self.assertTrue(item.live)
-
- self.gearman_server.hold_jobs_in_queue = False
- self.gearman_server.release()
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(C.data['status'], 'NEW')
- self.assertEqual(A.reported, 1)
- self.assertEqual(B.reported, 1)
- self.assertEqual(C.reported, 1)
-
- # Each job should have tested exactly one change
- for job in self.history:
- self.assertEqual(len(job.changes.split()), 1)
-
- @simple_layout('layouts/three-projects.yaml')
- def test_crd_check_transitive(self):
- "Test transitive cross-repo dependencies"
- # Specifically, if A -> B -> C, and C gets a new patchset and
- # A gets a new patchset, ensure the test of A,2 includes B,1
- # and C,2 (not C,1 which would indicate stale data in the
- # cache for B).
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
-
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- # B Depends-On: C
- B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- B.subject, C.data['id'])
-
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(self.history[-1].changes, '3,1 2,1 1,1')
-
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(self.history[-1].changes, '3,1 2,1')
-
- self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(self.history[-1].changes, '3,1')
-
- C.addPatchset()
- self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
- self.waitUntilSettled()
- self.assertEqual(self.history[-1].changes, '3,2')
-
- A.addPatchset()
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
- self.waitUntilSettled()
- self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
-
- def test_crd_check_unknown(self):
- "Test unknown projects in independent pipeline"
- self.init_repo("org/unknown", tag='init')
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'D')
- # A Depends-On: B
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
-
- # Make sure zuul has seen an event on B.
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- self.assertEqual(A.data['status'], 'NEW')
- self.assertEqual(A.reported, 1)
- self.assertEqual(B.data['status'], 'NEW')
- self.assertEqual(B.reported, 0)
-
- def test_crd_cycle_join(self):
- "Test an updated change creates a cycle"
- A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
-
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
- self.assertEqual(A.reported, 1)
-
- # Create B->A
- B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
- B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- B.subject, A.data['id'])
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- # Dep is there so zuul should have reported on B
- self.assertEqual(B.reported, 1)
-
- # Update A to add A->B (a cycle).
- A.addPatchset()
- A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
- A.subject, B.data['id'])
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
- self.waitUntilSettled()
-
- # Dependency cycle injected so zuul should not have reported again on A
- self.assertEqual(A.reported, 1)
-
- # Now if we update B to remove the depends-on, everything
- # should be okay. B; A->B
-
- B.addPatchset()
- B.data['commitMessage'] = '%s\n' % (B.subject,)
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
- self.waitUntilSettled()
-
- # Cycle was removed so now zuul should have reported again on A
- self.assertEqual(A.reported, 2)
-
- self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
- self.waitUntilSettled()
- self.assertEqual(B.reported, 2)
-
@simple_layout('layouts/disable_at.yaml')
def test_disable_at(self):
"Test a pipeline will only report to the disabled trigger when failing"
diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py
index 59dd8b0..b999106 100644
--- a/tests/unit/test_streaming.py
+++ b/tests/unit/test_streaming.py
@@ -41,13 +41,13 @@
def startStreamer(self, port, root=None):
if not root:
root = tempfile.gettempdir()
- return zuul.lib.log_streamer.LogStreamer(None, self.host, port, root)
+ return zuul.lib.log_streamer.LogStreamer(self.host, port, root)
def test_start_stop(self):
- port = 7900
- streamer = self.startStreamer(port)
+ streamer = self.startStreamer(0)
self.addCleanup(streamer.stop)
+ port = streamer.server.socket.getsockname()[1]
s = socket.create_connection((self.host, port))
s.close()
@@ -77,8 +77,9 @@
def startStreamer(self, port, build_uuid, root=None):
if not root:
root = tempfile.gettempdir()
- self.streamer = zuul.lib.log_streamer.LogStreamer(None, self.host,
+ self.streamer = zuul.lib.log_streamer.LogStreamer(self.host,
port, root)
+ port = self.streamer.server.socket.getsockname()[1]
s = socket.create_connection((self.host, port))
self.addCleanup(s.close)
@@ -129,10 +130,9 @@
# Create a thread to stream the log. We need this to be happening
# before we create the flag file to tell the job to complete.
- port = 7901
streamer_thread = threading.Thread(
target=self.startStreamer,
- args=(port, build.uuid, self.executor_server.jobdir_root,)
+ args=(0, build.uuid, self.executor_server.jobdir_root,)
)
streamer_thread.start()
self.addCleanup(self.stopStreamer)
@@ -209,7 +209,7 @@
def test_websocket_streaming(self):
# Start the finger streamer daemon
streamer = zuul.lib.log_streamer.LogStreamer(
- None, self.host, 0, self.executor_server.jobdir_root)
+ self.host, 0, self.executor_server.jobdir_root)
self.addCleanup(streamer.stop)
# Need to set the streaming port before submitting the job
@@ -294,7 +294,7 @@
def test_finger_gateway(self):
# Start the finger streamer daemon
streamer = zuul.lib.log_streamer.LogStreamer(
- None, self.host, 0, self.executor_server.jobdir_root)
+ self.host, 0, self.executor_server.jobdir_root)
self.addCleanup(streamer.stop)
finger_port = streamer.server.socket.getsockname()[1]
diff --git a/tests/unit/test_zuultrigger.py b/tests/unit/test_zuultrigger.py
index 3954a21..5575853 100644
--- a/tests/unit/test_zuultrigger.py
+++ b/tests/unit/test_zuultrigger.py
@@ -126,5 +126,5 @@
"dependencies was unable to be automatically merged with the "
"current state of its repository. Please rebase the change and "
"upload a new patchset.")
- self.assertEqual(self.fake_gerrit.queries[1],
- "project:org/project status:open")
+ self.assertIn("project:org/project status:open",
+ self.fake_gerrit.queries)
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index ade9715..ad7aaa8 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -14,10 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import grp
import logging
import os
-import pwd
import sys
import signal
import tempfile
@@ -64,7 +62,7 @@
self.log.info("Starting log streamer")
streamer = zuul.lib.log_streamer.LogStreamer(
- self.user, '::', self.finger_port, self.job_dir)
+ '::', self.finger_port, self.job_dir)
# Keep running until the parent dies:
pipe_read = os.fdopen(pipe_read)
@@ -76,22 +74,6 @@
os.close(pipe_read)
self.log_streamer_pid = child_pid
- def change_privs(self):
- '''
- Drop our privileges to the zuul user.
- '''
- if os.getuid() != 0:
- return
- pw = pwd.getpwnam(self.user)
- # get a list of supplementary groups for the target user, and make sure
- # we set them when dropping privileges.
- groups = [g.gr_gid for g in grp.getgrall() if self.user in g.gr_mem]
- os.setgroups(groups)
- os.setgid(pw.pw_gid)
- os.setuid(pw.pw_uid)
- os.chdir(pw.pw_dir)
- os.umask(0o022)
-
def run(self):
if self.args.command in zuul.executor.server.COMMANDS:
self.send_command(self.args.command)
@@ -99,8 +81,6 @@
self.configure_connections(source_only=True)
- self.user = get_default(self.config, 'executor', 'user', 'zuul')
-
if self.config.has_option('executor', 'job_dir'):
self.job_dir = os.path.expanduser(
self.config.get('executor', 'job_dir'))
@@ -120,7 +100,6 @@
)
self.start_log_streamer()
- self.change_privs()
ExecutorServer = zuul.executor.server.ExecutorServer
self.executor = ExecutorServer(self.config, self.connections,
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index f4b090d..d3b3c00 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -442,8 +442,19 @@
# In case this change is already in the history we have a
# cyclic dependency and don't need to update ourselves again
# as this gets done in a previous frame of the call stack.
- # NOTE(jeblair): I don't think it's possible to hit this case
- # anymore as all paths hit the change cache first.
+ # NOTE(jeblair): The only case where this can still be hit is
+ # when we get an event for a change with no associated
+ # patchset; for instance, when the gerrit topic is changed.
+ # In that case, we will update change 1234,None, which will be
+ # inserted into the cache as its own entry, but then we will
+ # resolve the patchset before adding it to the history list,
+ # then if there are dependencies, we can walk down and then
+ # back up to the version of this change with a patchset which
+ # will match the history list but will have bypassed the
+ # change cache because the previous object had a patchset of
+ # None. All paths hit the change cache first. To be able to
+ # drop history, we need to resolve the patchset on events with
+ # no patchsets before adding the entry to the change cache.
if (history and change.number and change.patchset and
(change.number, change.patchset) in history):
self.log.debug("Change %s is in history" % (change,))
@@ -461,6 +472,11 @@
change.project = self.source.getProject(data['project'])
change.branch = data['branch']
change.url = data['url']
+ change.uris = [
+ '%s/%s' % (self.server, change.number),
+ '%s/#/c/%s' % (self.server, change.number),
+ ]
+
max_ps = 0
files = []
for ps in data['patchSets']:
@@ -481,6 +497,7 @@
change.open = data['open']
change.status = data['status']
change.owner = data['owner']
+ change.message = data['commitMessage']
if change.is_merged:
# This change is merged, so we don't need to look any further
@@ -494,7 +511,8 @@
history = history[:]
history.append((change.number, change.patchset))
- needs_changes = []
+ needs_changes = set()
+ git_needs_changes = []
if 'dependsOn' in data:
parts = data['dependsOn'][0]['ref'].split('/')
dep_num, dep_ps = parts[3], parts[4]
@@ -505,8 +523,11 @@
# already merged. So even if it is "ABANDONED", we should not
# ignore it.
if (not dep.is_merged) and dep not in needs_changes:
- needs_changes.append(dep)
+ git_needs_changes.append(dep)
+ needs_changes.add(dep)
+ change.git_needs_changes = git_needs_changes
+ compat_needs_changes = []
for record in self._getDependsOnFromCommit(data['commitMessage'],
change):
dep_num = record['number']
@@ -516,10 +537,12 @@
(change, dep_num, dep_ps))
dep = self._getChange(dep_num, dep_ps, history=history)
if dep.open and dep not in needs_changes:
- needs_changes.append(dep)
- change.needs_changes = needs_changes
+ compat_needs_changes.append(dep)
+ needs_changes.add(dep)
+ change.compat_needs_changes = compat_needs_changes
- needed_by_changes = []
+ needed_by_changes = set()
+ git_needed_by_changes = []
if 'neededBy' in data:
for needed in data['neededBy']:
parts = needed['ref'].split('/')
@@ -527,9 +550,13 @@
self.log.debug("Updating %s: Getting git-needed change %s,%s" %
(change, dep_num, dep_ps))
dep = self._getChange(dep_num, dep_ps, history=history)
- if dep.open and dep.is_current_patchset:
- needed_by_changes.append(dep)
+ if (dep.open and dep.is_current_patchset and
+ dep not in needed_by_changes):
+ git_needed_by_changes.append(dep)
+ needed_by_changes.add(dep)
+ change.git_needed_by_changes = git_needed_by_changes
+ compat_needed_by_changes = []
for record in self._getNeededByFromCommit(data['id'], change):
dep_num = record['number']
dep_ps = record['currentPatchSet']['number']
@@ -543,9 +570,13 @@
refresh = (dep_num, dep_ps) not in history
dep = self._getChange(
dep_num, dep_ps, refresh=refresh, history=history)
- if dep.open and dep.is_current_patchset:
- needed_by_changes.append(dep)
- change.needed_by_changes = needed_by_changes
+ if (dep.open and dep.is_current_patchset
+ and dep not in needed_by_changes):
+ compat_needed_by_changes.append(dep)
+ needed_by_changes.add(dep)
+ change.compat_needed_by_changes = compat_needed_by_changes
+
+ self.sched.onChangeUpdated(change)
return change
diff --git a/zuul/driver/gerrit/gerritsource.py b/zuul/driver/gerrit/gerritsource.py
index 7141080..9e327b9 100644
--- a/zuul/driver/gerrit/gerritsource.py
+++ b/zuul/driver/gerrit/gerritsource.py
@@ -12,12 +12,15 @@
# License for the specific language governing permissions and limitations
# under the License.
+import re
+import urllib
import logging
import voluptuous as vs
from zuul.source import BaseSource
from zuul.model import Project
from zuul.driver.gerrit.gerritmodel import GerritRefFilter
from zuul.driver.util import scalar_or_list, to_list
+from zuul.lib.dependson import find_dependency_headers
class GerritSource(BaseSource):
@@ -44,6 +47,61 @@
def getChange(self, event, refresh=False):
return self.connection.getChange(event, refresh)
+ change_re = re.compile(r"/(\#\/c\/)?(\d+)[\w]*")
+
+ def getChangeByURL(self, url):
+ try:
+ parsed = urllib.parse.urlparse(url)
+ except ValueError:
+ return None
+ m = self.change_re.match(parsed.path)
+ if not m:
+ return None
+ try:
+ change_no = int(m.group(2))
+ except ValueError:
+ return None
+ query = "change:%s" % (change_no,)
+ results = self.connection.simpleQuery(query)
+ if not results:
+ return None
+ change = self.connection._getChange(
+ results[0]['number'], results[0]['currentPatchSet']['number'])
+ return change
+
+ def getChangesDependingOn(self, change, projects):
+ changes = []
+ if not change.uris:
+ return changes
+ queries = set()
+ for uri in change.uris:
+ queries.add('message:%s' % uri)
+ query = '(' + ' OR '.join(queries) + ')'
+ results = self.connection.simpleQuery(query)
+ seen = set()
+ for result in results:
+ for match in find_dependency_headers(result['commitMessage']):
+ found = False
+ for uri in change.uris:
+ if uri in match:
+ found = True
+ break
+ if not found:
+ continue
+ key = (result['number'], result['currentPatchSet']['number'])
+ if key in seen:
+ continue
+ seen.add(key)
+ change = self.connection._getChange(
+ result['number'], result['currentPatchSet']['number'])
+ changes.append(change)
+ return changes
+
+ def getCachedChanges(self):
+ for x in self.connection._change_cache.values():
+ for y in x.values():
+ yield y
+
def getProject(self, name):
p = self.connection.getProject(name)
if not p:
diff --git a/zuul/driver/git/gitsource.py b/zuul/driver/git/gitsource.py
index 78ae04e..a7d42be 100644
--- a/zuul/driver/git/gitsource.py
+++ b/zuul/driver/git/gitsource.py
@@ -38,6 +38,15 @@
def getChange(self, event, refresh=False):
return self.connection.getChange(event, refresh)
+ def getChangeByURL(self, url):
+ return None
+
+ def getChangesDependingOn(self, change, projects):
+ return []
+
+ def getCachedChanges(self):
+ return []
+
def getProject(self, name):
p = self.connection.getProject(name)
if not p:
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 4b91c18..a7aefe0 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -646,9 +646,12 @@
return self._github
def maintainCache(self, relevant):
+ remove = set()
for key, change in self._change_cache.items():
if change not in relevant:
- del self._change_cache[key]
+ remove.add(key)
+ for key in remove:
+ del self._change_cache[key]
def getChange(self, event, refresh=False):
"""Get the change representing an event."""
@@ -658,7 +661,9 @@
change = self._getChange(project, event.change_number,
event.patch_number, refresh=refresh)
change.url = event.change_url
- change.updated_at = self._ghTimestampToDate(event.updated_at)
+ change.uris = [
+ '%s/%s/pull/%s' % (self.server, project, change.number),
+ ]
change.source_event = event
change.is_current_patchset = (change.pr.get('head').get('sha') ==
event.patch_number)
@@ -699,58 +704,72 @@
raise
return change
- def _getDependsOnFromPR(self, body):
- prs = []
- seen = set()
+ def getChangesDependingOn(self, change, projects):
+ changes = []
+ if not change.uris:
+ return changes
- for match in self.depends_on_re.findall(body):
- if match in seen:
- self.log.debug("Ignoring duplicate Depends-On: %s" % (match,))
- continue
- seen.add(match)
- # Get the github url
- url = match.rsplit()[-1]
- # break it into the parts we need
- _, org, proj, _, num = url.rsplit('/', 4)
- # Get a pull object so we can get the head sha
- pull = self.getPull('%s/%s' % (org, proj), int(num))
- prs.append(pull)
+ # Get a list of projects with unique installation ids
+ installation_ids = set()
+ installation_projects = set()
- return prs
+ if projects:
+ # We only need to find changes in projects in the supplied
+ # ChangeQueue. Find all of the github installations for
+ # all of those projects, and search using each of them, so
+ # that if we get the right results based on the
+ # permissions granted to each of the installations. The
+ # common case for this is likely to be just one
+ # installation -- change queues aren't likely to span more
+ # than one installation.
+ for project in projects:
+ installation_id = self.installation_map.get(project)
+ if installation_id not in installation_ids:
+ installation_ids.add(installation_id)
+ installation_projects.add(project)
+ else:
+ # We aren't in the context of a change queue and we just
+ # need to query all installations. This currently only
+ # happens if certain features of the zuul trigger are
+ # used; generally it should be avoided.
+ for project, installation_id in self.installation_map.items():
+ if installation_id not in installation_ids:
+ installation_ids.add(installation_id)
+ installation_projects.add(project)
- def _getNeededByFromPR(self, change):
- prs = []
- seen = set()
- # This shouldn't return duplicate issues, but code as if it could
-
- # This leaves off the protocol, but looks for the specific GitHub
- # hostname, the org/project, and the pull request number.
- pattern = 'Depends-On %s/%s/pull/%s' % (self.server,
- change.project.name,
- change.number)
+ keys = set()
+ pattern = ' OR '.join(change.uris)
query = '%s type:pr is:open in:body' % pattern
- # FIXME(tobiash): find a way to query this for different installations
- github = self.getGithubClient(change.project.name)
- for issue in github.search_issues(query=query):
- pr = issue.issue.pull_request().as_dict()
- if not pr.get('url'):
- continue
- if issue in seen:
- continue
- # the issue provides no good description of the project :\
- org, proj, _, num = pr.get('url').split('/')[-4:]
- self.log.debug("Found PR %s/%s/%s needs %s/%s" %
- (org, proj, num, change.project.name,
- change.number))
- prs.append(pr)
- seen.add(issue)
+ # Repeat the search for each installation id (project)
+ for installation_project in installation_projects:
+ github = self.getGithubClient(installation_project)
+ for issue in github.search_issues(query=query):
+ pr = issue.issue.pull_request().as_dict()
+ if not pr.get('url'):
+ continue
+ # the issue provides no good description of the project :\
+ org, proj, _, num = pr.get('url').split('/')[-4:]
+ proj = pr.get('base').get('repo').get('full_name')
+ sha = pr.get('head').get('sha')
+ key = (proj, num, sha)
+ if key in keys:
+ continue
+ self.log.debug("Found PR %s/%s needs %s/%s" %
+ (proj, num, change.project.name,
+ change.number))
+ keys.add(key)
+ self.log.debug("Ran search issues: %s", query)
+ log_rate_limit(self.log, github)
- self.log.debug("Ran search issues: %s", query)
- log_rate_limit(self.log, github)
- return prs
+ for key in keys:
+ (proj, num, sha) = key
+ project = self.source.getProject(proj)
+ change = self._getChange(project, int(num), patchset=sha)
+ changes.append(change)
+
+ return changes
def _updateChange(self, change, history=None):
-
# If this change is already in the history, we have a cyclic
# dependency loop and we do not need to update again, since it
# was done in a previous frame.
@@ -770,10 +789,10 @@
change.reviews = self.getPullReviews(change.project,
change.number)
change.labels = change.pr.get('labels')
- change.body = change.pr.get('body')
- # ensure body is at least an empty string
- if not change.body:
- change.body = ''
+ # ensure message is at least an empty string
+ change.message = change.pr.get('body') or ''
+ change.updated_at = self._ghTimestampToDate(
+ change.pr.get('updated_at'))
if history is None:
history = []
@@ -781,38 +800,7 @@
history = history[:]
history.append((change.project.name, change.number))
- needs_changes = []
-
- # Get all the PRs this may depend on
- for pr in self._getDependsOnFromPR(change.body):
- proj = pr.get('base').get('repo').get('full_name')
- pull = pr.get('number')
- self.log.debug("Updating %s: Getting dependent "
- "pull request %s/%s" %
- (change, proj, pull))
- project = self.source.getProject(proj)
- dep = self._getChange(project, pull,
- patchset=pr.get('head').get('sha'),
- history=history)
- if (not dep.is_merged) and dep not in needs_changes:
- needs_changes.append(dep)
-
- change.needs_changes = needs_changes
-
- needed_by_changes = []
- for pr in self._getNeededByFromPR(change):
- proj = pr.get('base').get('repo').get('full_name')
- pull = pr.get('number')
- self.log.debug("Updating %s: Getting needed "
- "pull request %s/%s" %
- (change, proj, pull))
- project = self.source.getProject(proj)
- dep = self._getChange(project, pull,
- patchset=pr.get('head').get('sha'),
- history=history)
- if not dep.is_merged:
- needed_by_changes.append(dep)
- change.needed_by_changes = needed_by_changes
+ self.sched.onChangeUpdated(change)
return change
diff --git a/zuul/driver/github/githubmodel.py b/zuul/driver/github/githubmodel.py
index ffd1c3f..0731dd7 100644
--- a/zuul/driver/github/githubmodel.py
+++ b/zuul/driver/github/githubmodel.py
@@ -37,7 +37,8 @@
self.labels = []
def isUpdateOf(self, other):
- if (hasattr(other, 'number') and self.number == other.number and
+ if (self.project == other.project and
+ hasattr(other, 'number') and self.number == other.number and
hasattr(other, 'patchset') and self.patchset != other.patchset and
hasattr(other, 'updated_at') and
self.updated_at > other.updated_at):
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
index 1e7e07a..33f8f7c 100644
--- a/zuul/driver/github/githubsource.py
+++ b/zuul/driver/github/githubsource.py
@@ -12,6 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
+import re
+import urllib
import logging
import time
import voluptuous as v
@@ -44,6 +46,8 @@
if not change.number:
# Not a pull request, considering merged.
return True
+ # We don't need to perform another query because the API call
+ # to perform the merge will ensure this is updated.
return change.is_merged
def canMerge(self, change, allow_needs):
@@ -61,6 +65,38 @@
def getChange(self, event, refresh=False):
return self.connection.getChange(event, refresh)
+ change_re = re.compile(r"/(.*?)/(.*?)/pull/(\d+)[\w]*")
+
+ def getChangeByURL(self, url):
+ try:
+ parsed = urllib.parse.urlparse(url)
+ except ValueError:
+ return None
+ m = self.change_re.match(parsed.path)
+ if not m:
+ return None
+ org = m.group(1)
+ proj = m.group(2)
+ try:
+ num = int(m.group(3))
+ except ValueError:
+ return None
+ pull = self.connection.getPull('%s/%s' % (org, proj), int(num))
+ if not pull:
+ return None
+ proj = pull.get('base').get('repo').get('full_name')
+ project = self.getProject(proj)
+ change = self.connection._getChange(
+ project, num,
+ patchset=pull.get('head').get('sha'))
+ return change
+
+ def getChangesDependingOn(self, change, projects):
+ return self.connection.getChangesDependingOn(change, projects)
+
+ def getCachedChanges(self):
+ return self.connection._change_cache.values()
+
def getProject(self, name):
p = self.connection.getProject(name)
if not p:
diff --git a/zuul/driver/zuul/__init__.py b/zuul/driver/zuul/__init__.py
index 0f6ec7d..e381137 100644
--- a/zuul/driver/zuul/__init__.py
+++ b/zuul/driver/zuul/__init__.py
@@ -90,7 +90,18 @@
if not hasattr(change, 'needed_by_changes'):
self.log.debug(" %s does not support dependencies" % type(change))
return
- for needs in change.needed_by_changes:
+
+ # This is very inefficient, especially on systems with large
+ # numbers of github installations. This can be improved later
+ # with persistent storage of dependency information.
+ needed_by_changes = set(change.needed_by_changes)
+ for source in self.sched.connections.getSources():
+ self.log.debug(" Checking source: %s", source)
+ needed_by_changes.update(
+ source.getChangesDependingOn(change, None))
+ self.log.debug(" Following changes: %s", needed_by_changes)
+
+ for needs in needed_by_changes:
self._createParentChangeEnqueuedEvent(needs, pipeline)
def _createParentChangeEnqueuedEvent(self, change, pipeline):
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 06c2087..b21a290 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -245,7 +245,7 @@
for change in dependent_changes:
# We have to find the project this way because it may not
# be registered in the tenant (ie, a foreign project).
- source = self.sched.connections.getSourceByHostname(
+ source = self.sched.connections.getSourceByCanonicalHostname(
change['project']['canonical_hostname'])
project = source.getProject(change['project']['name'])
if project not in projects:
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 5a710a6..a8ab8c4 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -44,7 +44,8 @@
BUFFER_LINES_FOR_SYNTAX = 200
COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
'unverbose', 'keep', 'nokeep']
-DEFAULT_FINGER_PORT = 79
+DEFAULT_FINGER_PORT = 7900
+BLACKLISTED_ANSIBLE_CONNECTION_TYPES = ['network_cli']
class StopException(Exception):
@@ -347,6 +348,8 @@
pass
self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
+ self.setup_inventory = os.path.join(self.ansible_root,
+ 'setup-inventory.yaml')
self.logging_json = os.path.join(self.ansible_root, 'logging.json')
self.playbooks = [] # The list of candidate playbooks
self.playbook = None # A pointer to the candidate we have chosen
@@ -493,6 +496,26 @@
shutil.copy(os.path.join(library_path, fn), target_dir)
+def make_setup_inventory_dict(nodes):
+
+ hosts = {}
+ for node in nodes:
+ if (node['host_vars']['ansible_connection'] in
+ BLACKLISTED_ANSIBLE_CONNECTION_TYPES):
+ continue
+
+ for name in node['name']:
+ hosts[name] = node['host_vars']
+
+ inventory = {
+ 'all': {
+ 'hosts': hosts,
+ }
+ }
+
+ return inventory
+
+
def make_inventory_dict(nodes, groups, all_vars):
hosts = {}
@@ -1157,8 +1180,13 @@
result_data_file=self.jobdir.result_data_file)
nodes = self.getHostList(args)
+ setup_inventory = make_setup_inventory_dict(nodes)
inventory = make_inventory_dict(nodes, args['groups'], all_vars)
+ with open(self.jobdir.setup_inventory, 'w') as setup_inventory_yaml:
+ setup_inventory_yaml.write(
+ yaml.safe_dump(setup_inventory, default_flow_style=False))
+
with open(self.jobdir.inventory, 'w') as inventory_yaml:
inventory_yaml.write(
yaml.safe_dump(inventory, default_flow_style=False))
@@ -1423,6 +1451,7 @@
verbose = '-v'
cmd = ['ansible', '*', verbose, '-m', 'setup',
+ '-i', self.jobdir.setup_inventory,
'-a', 'gather_subset=!all']
result, code = self.runAnsible(
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 262490a..33c66f9 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -14,6 +14,7 @@
import logging
import re
+from collections import OrderedDict
import zuul.driver.zuul
import zuul.driver.gerrit
@@ -38,7 +39,7 @@
log = logging.getLogger("zuul.ConnectionRegistry")
def __init__(self):
- self.connections = {}
+ self.connections = OrderedDict()
self.drivers = {}
self.registerDriver(zuul.driver.zuul.ZuulDriver())
@@ -85,7 +86,7 @@
def configure(self, config, source_only=False):
# Register connections from the config
- connections = {}
+ connections = OrderedDict()
for section_name in config.sections():
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
@@ -154,6 +155,13 @@
connection = self.connections[connection_name]
return connection.driver.getSource(connection)
+ def getSources(self):
+ sources = []
+ for connection in self.connections.values():
+ if hasattr(connection.driver, 'getSource'):
+ sources.append(connection.driver.getSource(connection))
+ return sources
+
def getReporter(self, connection_name, config=None):
connection = self.connections[connection_name]
return connection.driver.getReporter(connection, config)
@@ -162,7 +170,7 @@
connection = self.connections[connection_name]
return connection.driver.getTrigger(connection, config)
- def getSourceByHostname(self, canonical_hostname):
+ def getSourceByCanonicalHostname(self, canonical_hostname):
for connection in self.connections.values():
if hasattr(connection, 'canonical_hostname'):
if connection.canonical_hostname == canonical_hostname:
diff --git a/zuul/lib/dependson.py b/zuul/lib/dependson.py
new file mode 100644
index 0000000..cd0f6ef
--- /dev/null
+++ b/zuul/lib/dependson.py
@@ -0,0 +1,29 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+
+
+DEPENDS_ON_RE = re.compile(r"^Depends-On: (.*?)\s*$",
+ re.MULTILINE | re.IGNORECASE)
+
+
+def find_dependency_headers(message):
+ # Search for Depends-On headers
+ dependencies = []
+ for match in DEPENDS_ON_RE.findall(message):
+ if match in dependencies:
+ continue
+ dependencies.append(match)
+ return dependencies
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index c778812..f96f442 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -157,12 +157,11 @@
Class implementing log streaming over the finger daemon port.
'''
- def __init__(self, user, host, port, jobdir_root):
+ def __init__(self, host, port, jobdir_root):
self.log = logging.getLogger('zuul.log_streamer')
self.log.debug("LogStreamer starting on port %s", port)
self.server = LogStreamerServer((host, port),
RequestHandler,
- user=user,
jobdir_root=jobdir_root)
# We start the actual serving within a thread so we can return to
diff --git a/zuul/lib/streamer_utils.py b/zuul/lib/streamer_utils.py
index 43bc286..3d2d561 100644
--- a/zuul/lib/streamer_utils.py
+++ b/zuul/lib/streamer_utils.py
@@ -74,7 +74,7 @@
address_family = socket.AF_INET6
def __init__(self, *args, **kwargs):
- self.user = kwargs.pop('user')
+ self.user = kwargs.pop('user', None)
self.pid_file = kwargs.pop('pid_file', None)
socketserver.ThreadingTCPServer.__init__(self, *args, **kwargs)
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index d205afc..b8a280f 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -12,9 +12,11 @@
import logging
import textwrap
+import urllib
from zuul import exceptions
from zuul import model
+from zuul.lib.dependson import find_dependency_headers
class DynamicChangeQueueContextManager(object):
@@ -343,6 +345,32 @@
self.dequeueItem(item)
self.reportStats(item)
+ def updateCommitDependencies(self, change, change_queue):
+ # Search for Depends-On headers and find appropriate changes
+ self.log.debug(" Updating commit dependencies for %s", change)
+ change.refresh_deps = False
+ dependencies = []
+ seen = set()
+ for match in find_dependency_headers(change.message):
+ self.log.debug(" Found Depends-On header: %s", match)
+ if match in seen:
+ continue
+ seen.add(match)
+ try:
+ url = urllib.parse.urlparse(match)
+ except ValueError:
+ continue
+ source = self.sched.connections.getSourceByCanonicalHostname(
+ url.hostname)
+ if not source:
+ continue
+ self.log.debug(" Found source: %s", source)
+ dep = source.getChangeByURL(match)
+ if dep and (not dep.is_merged) and dep not in dependencies:
+ self.log.debug(" Adding dependency: %s", dep)
+ dependencies.append(dep)
+ change.commit_needs_changes = dependencies
+
def provisionNodes(self, item):
jobs = item.findJobsToRequest()
if not jobs:
diff --git a/zuul/manager/dependent.py b/zuul/manager/dependent.py
index 5aef453..20b376d 100644
--- a/zuul/manager/dependent.py
+++ b/zuul/manager/dependent.py
@@ -95,12 +95,29 @@
def enqueueChangesBehind(self, change, quiet, ignore_requirements,
change_queue):
self.log.debug("Checking for changes needing %s:" % change)
- to_enqueue = []
- source = change.project.source
if not hasattr(change, 'needed_by_changes'):
self.log.debug(" %s does not support dependencies" % type(change))
return
- for other_change in change.needed_by_changes:
+
+ # for project in change_queue, project.source get changes, then dedup.
+ sources = set()
+ for project in change_queue.projects:
+ sources.add(project.source)
+
+ seen = set(change.needed_by_changes)
+ needed_by_changes = change.needed_by_changes[:]
+ for source in sources:
+ self.log.debug(" Checking source: %s", source)
+ for c in source.getChangesDependingOn(change,
+ change_queue.projects):
+ if c not in seen:
+ seen.add(c)
+ needed_by_changes.append(c)
+
+ self.log.debug(" Following changes: %s", needed_by_changes)
+
+ to_enqueue = []
+ for other_change in needed_by_changes:
with self.getChangeQueue(other_change) as other_change_queue:
if other_change_queue != change_queue:
self.log.debug(" Change %s in project %s can not be "
@@ -108,6 +125,7 @@
(other_change, other_change.project,
change_queue))
continue
+ source = other_change.project.source
if source.canMerge(other_change, self.getSubmitAllowNeeds()):
self.log.debug(" Change %s needs %s and is ready to merge" %
(other_change, change))
@@ -145,10 +163,12 @@
return True
def checkForChangesNeededBy(self, change, change_queue):
- self.log.debug("Checking for changes needed by %s:" % change)
- source = change.project.source
# Return true if okay to proceed enqueing this change,
# false if the change should not be enqueued.
+ self.log.debug("Checking for changes needed by %s:" % change)
+ if (hasattr(change, 'commit_needs_changes') and
+ (change.refresh_deps or change.commit_needs_changes is None)):
+ self.updateCommitDependencies(change, change_queue)
if not hasattr(change, 'needs_changes'):
self.log.debug(" %s does not support dependencies" % type(change))
return True
@@ -180,7 +200,8 @@
self.log.debug(" Needed change is already ahead "
"in the queue")
continue
- if source.canMerge(needed_change, self.getSubmitAllowNeeds()):
+ if needed_change.project.source.canMerge(
+ needed_change, self.getSubmitAllowNeeds()):
self.log.debug(" Change %s is needed" % needed_change)
if needed_change not in changes_needed:
changes_needed.append(needed_change)
diff --git a/zuul/manager/independent.py b/zuul/manager/independent.py
index 65f5ca0..0c2baf0 100644
--- a/zuul/manager/independent.py
+++ b/zuul/manager/independent.py
@@ -70,6 +70,9 @@
self.log.debug("Checking for changes needed by %s:" % change)
# Return true if okay to proceed enqueing this change,
# false if the change should not be enqueued.
+ if (hasattr(change, 'commit_needs_changes') and
+ (change.refresh_deps or change.commit_needs_changes is None)):
+ self.updateCommitDependencies(change, None)
if not hasattr(change, 'needs_changes'):
self.log.debug(" %s does not support dependencies" % type(change))
return True
diff --git a/zuul/model.py b/zuul/model.py
index 16a701d..bac9e4c 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -2103,11 +2103,28 @@
def __init__(self, project):
super(Change, self).__init__(project)
self.number = None
+ # The gitweb url for browsing the change
self.url = None
+ # URIs for this change which may appear in depends-on headers.
+ # Note this omits the scheme; i.e., is hostname/path.
+ self.uris = []
self.patchset = None
- self.needs_changes = []
- self.needed_by_changes = []
+ # Changes that the source determined are needed due to the
+ # git DAG:
+ self.git_needs_changes = []
+ self.git_needed_by_changes = []
+
+ # Changes that the source determined are needed by backwards
+ # compatible processing of Depends-On headers (Gerrit only):
+ self.compat_needs_changes = []
+ self.compat_needed_by_changes = []
+
+ # Changes that the pipeline manager determined are needed due
+ # to Depends-On headers (all drivers):
+ self.commit_needs_changes = None
+ self.refresh_deps = False
+
self.is_current_patchset = True
self.can_merge = False
self.is_merged = False
@@ -2116,6 +2133,11 @@
self.status = None
self.owner = None
+ # This may be the commit message, or it may be a cover message
+ # in the case of a PR. Either way, it's the place where we
+ # look for depends-on headers.
+ self.message = None
+
self.source_event = None
def _id(self):
@@ -2129,8 +2151,18 @@
return True
return False
+ @property
+ def needs_changes(self):
+ return (self.git_needs_changes + self.compat_needs_changes +
+ self.commit_needs_changes)
+
+ @property
+ def needed_by_changes(self):
+ return (self.git_needed_by_changes + self.compat_needed_by_changes)
+
def isUpdateOf(self, other):
- if ((hasattr(other, 'number') and self.number == other.number) and
+ if (self.project == other.project and
+ (hasattr(other, 'number') and self.number == other.number) and
(hasattr(other, 'patchset') and
self.patchset is not None and
other.patchset is not None and
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index c3f2f23..a2e3b6e 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -1088,3 +1088,25 @@
for pipeline in tenant.layout.pipelines.values():
pipelines.append(pipeline.formatStatusJSON(websocket_url))
return json.dumps(data)
+
+ def onChangeUpdated(self, change):
+ """Remove stale dependency references on change update.
+
+ When a change is updated with a new patchset, other changes in
+ the system may still have a reference to the old patchset in
+ their dependencies. Search for those (across all sources) and
+ mark that their dependencies are out of date. This will cause
+ them to be refreshed the next time the queue processor
+ examines them.
+ """
+
+ self.log.debug("Change %s has been updated, clearing dependent "
+ "change caches", change)
+ for source in self.connections.getSources():
+ for other_change in source.getCachedChanges():
+ if other_change.commit_needs_changes is None:
+ continue
+ for dep in other_change.commit_needs_changes:
+ if change.isUpdateOf(dep):
+ other_change.refresh_deps = True
+ change.refresh_deps = True
diff --git a/zuul/source/__init__.py b/zuul/source/__init__.py
index 0396aff..00dfc9c 100644
--- a/zuul/source/__init__.py
+++ b/zuul/source/__init__.py
@@ -52,6 +52,29 @@
"""Get the change representing an event."""
@abc.abstractmethod
+ def getChangeByURL(self, url):
+ """Get the change corresponding to the supplied URL.
+
+ The URL may may not correspond to this source; if it doesn't,
+ or there is no change at that URL, return None.
+
+ """
+
+ @abc.abstractmethod
+ def getChangesDependingOn(self, change, projects):
+ """Return changes which depend on changes at the supplied URIs.
+
+ Search this source for changes which depend on the supplied
+ change. Generally the Change.uris attribute should be used to
+ perform the search, as it contains a list of URLs without the
+ scheme which represent a single change
+
+ If the projects argument is None, search across all known
+ projects. If it is supplied, the search may optionally be
+ restricted to only those projects.
+ """
+
+ @abc.abstractmethod
def getProjectOpenChanges(self, project):
"""Get the open changes for a project."""