Merge "ansible: honor command no_log module attribute"
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 525cb38..e0526f5 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -1137,8 +1137,12 @@
encrypted, however, data which are not sensitive may be provided
unencrypted as well for convenience.
-A Secret may only be used by jobs defined within the same project. To
-use a secret, a :ref:`job` must specify the secret in
+A Secret may only be used by jobs defined within the same project.
+Note that they can be used by any branch of that project, so if a
+project's branches have different access controls, consider whether
+all branches of that project are equally trusted before using secrets.
+
+To use a secret, a :ref:`job` must specify the secret in
:attr:`job.secrets`. Secrets are bound to the playbooks associated
with the specific job definition where they were declared. Additional
pre or post playbooks which appear in child jobs will not have access
@@ -1175,6 +1179,12 @@
`allowed-projects` job attribute can be used to restrict the projects
which can invoke that job.
+Secrets, like most configuration items, are unique within a tenant,
+though a secret may be defined on multiple branches of the same
+project as long as the contents are the same. This is to aid in
+branch maintenance, so that creating a new branch based on an existing
+branch will not immediately produce a configuration error.
+
.. attr:: secret
The following attributes must appear on a secret:
@@ -1203,6 +1213,12 @@
groups of node types once and referring to them by name, job
configuration may be simplified.
+Nodesets, like most configuration items, are unique within a tenant,
+though a nodeset may be defined on multiple branches of the same
+project as long as the contents are the same. This is to aid in
+branch maintenance, so that creating a new branch based on an existing
+branch will not immediately produce a configuration error.
+
.. code-block:: yaml
- nodeset:
@@ -1285,9 +1301,19 @@
represents the maximum number of jobs which use that semaphore at the
same time.
+Semaphores, like most configuration items, are unique within a tenant,
+though a semaphore may be defined on multiple branches of the same
+project as long as the value is the same. This is to aid in branch
+maintenance, so that creating a new branch based on an existing branch
+will not immediately produce a configuration error.
+
Semaphores are never subject to dynamic reconfiguration. If the value
of a semaphore is changed, it will take effect only when the change
-where it is updated is merged. An example follows:
+where it is updated is merged. However, Zuul will attempt to validate
+the configuration of semaphores in proposed updates, even if they
+aren't used.
+
+An example usage of semaphores follows:
.. code-block:: yaml
diff --git a/tests/base.py b/tests/base.py
index c449242..fe01399 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -606,7 +606,7 @@
return ret
def getGitUrl(self, project):
- return os.path.join(self.upstream_root, project.name)
+ return 'file://' + os.path.join(self.upstream_root, project.name)
class GithubChangeReference(git.Reference):
@@ -1794,18 +1794,6 @@
else:
self._log_stream = sys.stdout
- # NOTE(jeblair): this is temporary extra debugging to try to
- # track down a possible leak.
- orig_git_repo_init = git.Repo.__init__
-
- def git_repo_init(myself, *args, **kw):
- orig_git_repo_init(myself, *args, **kw)
- self.log.debug("Created git repo 0x%x %s" %
- (id(myself), repr(myself)))
-
- self.useFixture(fixtures.MonkeyPatch('git.Repo.__init__',
- git_repo_init))
-
handler = logging.StreamHandler(self._log_stream)
formatter = logging.Formatter('%(asctime)s %(name)-32s '
'%(levelname)-8s %(message)s')
@@ -1960,6 +1948,9 @@
self.config.set(
'executor', 'command_socket',
os.path.join(self.test_root, 'executor.socket'))
+ self.config.set(
+ 'merger', 'command_socket',
+ os.path.join(self.test_root, 'merger.socket'))
self.statsd = FakeStatsd()
if self.config.has_section('statsd'):
@@ -2016,6 +2007,7 @@
self.config, self.sched)
self.merge_client = zuul.merger.client.MergeClient(
self.config, self.sched)
+ self.merge_server = None
self.nodepool = zuul.nodepool.Nodepool(self.sched)
self.zk = zuul.zk.ZooKeeper()
self.zk.connect(self.zk_config)
@@ -2290,6 +2282,8 @@
self.executor_server.release()
self.executor_client.stop()
self.merge_client.stop()
+ if self.merge_server:
+ self.merge_server.stop()
self.executor_server.stop()
self.sched.stop()
self.sched.join()
@@ -2362,6 +2356,13 @@
zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
+ def delete_branch(self, project, branch):
+ path = os.path.join(self.upstream_root, project)
+ repo = git.Repo(path)
+ repo.head.reference = repo.heads['master']
+ zuul.merger.merger.reset_repo_to_head(repo)
+ repo.delete_head(repo.heads[branch], force=True)
+
def create_commit(self, project):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
diff --git a/tests/fixtures/config/nodesets/git/common-config/playbooks/base.yaml b/tests/fixtures/config/nodesets/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/nodesets/git/common-config/zuul.yaml b/tests/fixtures/config/nodesets/git/common-config/zuul.yaml
new file mode 100644
index 0000000..e1e2fb7
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/common-config/zuul.yaml
@@ -0,0 +1,39 @@
+- pipeline:
+ name: check
+ manager: independent
+ post-review: true
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ start:
+ gerrit:
+ Verified: 0
+ precedence: high
+
+- job:
+ name: base
+ parent: null
+ run: playbooks/base.yaml
diff --git a/tests/fixtures/config/nodesets/git/org_project1/README b/tests/fixtures/config/nodesets/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/nodesets/git/org_project1/zuul.yaml b/tests/fixtures/config/nodesets/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..398269e
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project1/zuul.yaml
@@ -0,0 +1,15 @@
+- nodeset:
+ name: project1-nodeset
+ nodes:
+ - name: controller
+ label: ubuntu-xenial
+
+- job:
+ parent: base
+ name: project1-test
+ nodeset: project1-nodeset
+
+- project:
+ check:
+ jobs:
+ - project1-test
diff --git a/tests/fixtures/config/nodesets/git/org_project2/README b/tests/fixtures/config/nodesets/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/nodesets/git/org_project2/zuul-nodeset.yaml b/tests/fixtures/config/nodesets/git/org_project2/zuul-nodeset.yaml
new file mode 100644
index 0000000..cb969a7
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project2/zuul-nodeset.yaml
@@ -0,0 +1,18 @@
+- nodeset:
+ name: project2-nodeset
+ nodes:
+ name: controller
+ label: ubuntu-xenial
+
+- job:
+ parent: base
+ name: project2-test
+ nodeset: project2-nodeset
+
+- project:
+ check:
+ jobs:
+ - project2-test
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/nodesets/git/org_project2/zuul.yaml b/tests/fixtures/config/nodesets/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..a4b42b1
--- /dev/null
+++ b/tests/fixtures/config/nodesets/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+ parent: base
+ name: project2-test
+
+- project:
+ check:
+ jobs:
+ - project2-test
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/nodesets/main.yaml b/tests/fixtures/config/nodesets/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/nodesets/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
diff --git a/tests/fixtures/config/secrets/git/common-config/zuul.yaml b/tests/fixtures/config/secrets/git/common-config/zuul.yaml
new file mode 100644
index 0000000..f9dfacc
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/common-config/zuul.yaml
@@ -0,0 +1,38 @@
+- pipeline:
+ name: check
+ manager: independent
+ post-review: true
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ start:
+ gerrit:
+ Verified: 0
+ precedence: high
+
+- job:
+ name: base
+ parent: null
diff --git a/tests/fixtures/config/secrets/git/org_project1/README b/tests/fixtures/config/secrets/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/secrets/git/org_project1/playbooks/secret.yaml b/tests/fixtures/config/secrets/git/org_project1/playbooks/secret.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project1/playbooks/secret.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/secrets/git/org_project1/zuul.yaml b/tests/fixtures/config/secrets/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..f105ada
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project1/zuul.yaml
@@ -0,0 +1,26 @@
+- secret:
+ name: project1_secret
+ data:
+ username: test-username
+ password: !encrypted/pkcs1-oaep |
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+ ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+ 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+ Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+ xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+ aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+ +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
+- job:
+ parent: base
+ name: project1-secret
+ run: playbooks/secret.yaml
+ secrets:
+ - project1_secret
+
+- project:
+ check:
+ jobs:
+ - project1-secret
diff --git a/tests/fixtures/config/secrets/git/org_project2/README b/tests/fixtures/config/secrets/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/secrets/git/org_project2/playbooks/secret.yaml b/tests/fixtures/config/secrets/git/org_project2/playbooks/secret.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/playbooks/secret.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/secrets/git/org_project2/zuul-secret.yaml b/tests/fixtures/config/secrets/git/org_project2/zuul-secret.yaml
new file mode 100644
index 0000000..d6ffd47
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/zuul-secret.yaml
@@ -0,0 +1,29 @@
+- secret:
+ name: project2_secret
+ data:
+ username: test-username
+ password: !encrypted/pkcs1-oaep |
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+ ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+ 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+ Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+ xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+ aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+ +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
+- job:
+ parent: base
+ name: project2-secret
+ run: playbooks/secret.yaml
+ secrets:
+ - project2_secret
+
+- project:
+ check:
+ jobs:
+ - project2-secret
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/secrets/git/org_project2/zuul.yaml b/tests/fixtures/config/secrets/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..305a237
--- /dev/null
+++ b/tests/fixtures/config/secrets/git/org_project2/zuul.yaml
@@ -0,0 +1,12 @@
+- job:
+ parent: base
+ name: project2-secret
+ run: playbooks/secret.yaml
+
+- project:
+ check:
+ jobs:
+ - project2-secret
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/secrets/main.yaml b/tests/fixtures/config/secrets/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/secrets/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
diff --git a/tests/fixtures/config/semaphore-branches/git/common-config/playbooks/base.yaml b/tests/fixtures/config/semaphore-branches/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/semaphore-branches/git/common-config/zuul.yaml b/tests/fixtures/config/semaphore-branches/git/common-config/zuul.yaml
new file mode 100644
index 0000000..e1e2fb7
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/common-config/zuul.yaml
@@ -0,0 +1,39 @@
+- pipeline:
+ name: check
+ manager: independent
+ post-review: true
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ start:
+ gerrit:
+ Verified: 0
+ precedence: high
+
+- job:
+ name: base
+ parent: null
+ run: playbooks/base.yaml
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project1/README b/tests/fixtures/config/semaphore-branches/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project1/zuul.yaml b/tests/fixtures/config/semaphore-branches/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..73766e0
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project1/zuul.yaml
@@ -0,0 +1,13 @@
+- semaphore:
+ name: project1-semaphore
+ max: 2
+
+- job:
+ parent: base
+ name: project1-test
+ semaphore: project1-semaphore
+
+- project:
+ check:
+ jobs:
+ - project1-test
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project2/README b/tests/fixtures/config/semaphore-branches/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project2/zuul-semaphore.yaml b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul-semaphore.yaml
new file mode 100644
index 0000000..db93fdb
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul-semaphore.yaml
@@ -0,0 +1,16 @@
+- semaphore:
+ name: project2-semaphore
+ max: 2
+
+- job:
+ parent: base
+ name: project2-test
+ semaphore: project2-semaphore
+
+- project:
+ check:
+ jobs:
+ - project2-test
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/semaphore-branches/git/org_project2/zuul.yaml b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..a4b42b1
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+ parent: base
+ name: project2-test
+
+- project:
+ check:
+ jobs:
+ - project2-test
+ gate:
+ jobs:
+ - noop
diff --git a/tests/fixtures/config/semaphore-branches/main.yaml b/tests/fixtures/config/semaphore-branches/main.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/semaphore-branches/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
diff --git a/tests/fixtures/layouts/branch-deletion.yaml b/tests/fixtures/layouts/branch-deletion.yaml
new file mode 100644
index 0000000..f72902a
--- /dev/null
+++ b/tests/fixtures/layouts/branch-deletion.yaml
@@ -0,0 +1,34 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ parent: null
+ run: playbooks/base.yaml
+
+- job:
+ name: project-test1
+ parent: base
+ branches: master
+
+- job:
+ name: project-test2
+ parent: base
+ branches: stable
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ - project-test2
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 5db20b3..3d76510 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -161,6 +161,44 @@
self.assertEqual(self.getJobFromHistory('project-test1').node,
'label2')
+ @simple_layout('layouts/branch-deletion.yaml')
+ def test_branch_deletion(self):
+ "Test the correct variant of a job runs on a branch"
+ self._startMerger()
+ for f in list(self.executor_server.merger_worker.functions.keys()):
+ f = str(f)
+ if f.startswith('merger:'):
+ self.executor_server.merger_worker.unRegisterFunction(f)
+
+ self.create_branch('org/project', 'stable')
+ A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-test2').result,
+ 'SUCCESS')
+
+ self.delete_branch('org/project', 'stable')
+ path = os.path.join(self.executor_src_root, 'review.example.com')
+ shutil.rmtree(path)
+
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ build = self.builds[0]
+
+ # Make sure there is no stable branch in the checked out git repo.
+ pname = 'review.example.com/org/project'
+ work = build.getWorkspaceRepos([pname])
+ work = work[pname]
+ heads = set([str(x) for x in work.heads])
+ self.assertEqual(heads, set(['master']))
+ self.executor_server.hold_jobs_in_build = False
+ build.release()
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+
def test_parallel_changes(self):
"Test that changes are tested in parallel and merged in series"
@@ -4934,7 +4972,7 @@
return repo_messages
def _test_merge(self, mode):
- us_path = os.path.join(
+ us_path = 'file://' + os.path.join(
self.upstream_root, 'org/project-%s' % mode)
expected_messages = [
'initial commit',
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 163a58b..4af5b47 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -2528,6 +2528,157 @@
self.assertHistory([])
+class TestSecrets(ZuulTestCase):
+ tenant_config_file = 'config/secrets/main.yaml'
+ secret = {'password': 'test-password',
+ 'username': 'test-username'}
+
+ def _getSecrets(self, job, pbtype):
+ secrets = []
+ build = self.getJobFromHistory(job)
+ for pb in build.parameters[pbtype]:
+ secrets.append(pb['secrets'])
+ return secrets
+
+ def test_secret_branch(self):
+ # Test that we can use a secret defined in another branch of
+ # the same project.
+ self.create_branch('org/project2', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project2', 'stable'))
+ self.waitUntilSettled()
+
+ with open(os.path.join(FIXTURE_DIR,
+ 'config/secrets/git/',
+ 'org_project2/zuul-secret.yaml')) as f:
+ config = f.read()
+
+ file_dict = {'zuul.yaml': config}
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ parent: base
+ name: project2-secret
+ run: playbooks/secret.yaml
+ secrets: [project2_secret]
+
+ - project:
+ check:
+ jobs:
+ - project2-secret
+ gate:
+ jobs:
+ - noop
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable', 'B',
+ files=file_dict)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 1, "B should report success")
+ self.assertHistory([
+ dict(name='project2-secret', result='SUCCESS', changes='2,1'),
+ ])
+ self.assertEqual(
+ self._getSecrets('project2-secret', 'playbooks'),
+ [{'project2_secret': self.secret}])
+
+ def test_secret_branch_duplicate(self):
+ # Test that we can create a duplicate secret on a different
+ # branch of the same project -- i.e., that when we branch
+ # master to stable on a project with a secret, nothing
+ # changes.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+ self.waitUntilSettled()
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1,
+ "A should report success")
+ self.assertHistory([
+ dict(name='project1-secret', result='SUCCESS', changes='1,1'),
+ ])
+ self.assertEqual(
+ self._getSecrets('project1-secret', 'playbooks'),
+ [{'project1_secret': self.secret}])
+
+ def test_secret_branch_error_same_branch(self):
+ # Test that we are unable to define a secret twice on the same
+ # project-branch.
+ in_repo_conf = textwrap.dedent(
+ """
+ - secret:
+ name: project1_secret
+ data: {}
+ - secret:
+ name: project1_secret
+ data: {}
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('already defined', A.messages[0])
+
+ def test_secret_branch_error_same_project(self):
+ # Test that we are unable to create a secret which differs
+ # from another with the same name -- i.e., that if we have a
+ # duplicate secret on multiple branches of the same project,
+ # they must be identical.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - secret:
+ name: project1_secret
+ data: {}
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('does not match existing definition in branch master',
+ A.messages[0])
+
+ def test_secret_branch_error_other_project(self):
+ # Test that we are unable to create a secret with the same
+ # name as another. We're never allowed to have a secret with
+ # the same name outside of a project.
+ in_repo_conf = textwrap.dedent(
+ """
+ - secret:
+ name: project1_secret
+ data: {}
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('already defined in project org/project1',
+ A.messages[0])
+
+
class TestSecretInheritance(ZuulTestCase):
tenant_config_file = 'config/secret-inheritance/main.yaml'
@@ -2724,6 +2875,278 @@
self._test_secret_file_fail()
+class TestNodesets(ZuulTestCase):
+ tenant_config_file = 'config/nodesets/main.yaml'
+
+ def test_nodeset_branch(self):
+ # Test that we can use a nodeset defined in another branch of
+ # the same project.
+ self.create_branch('org/project2', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project2', 'stable'))
+ self.waitUntilSettled()
+
+ with open(os.path.join(FIXTURE_DIR,
+ 'config/nodesets/git/',
+ 'org_project2/zuul-nodeset.yaml')) as f:
+ config = f.read()
+
+ file_dict = {'zuul.yaml': config}
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ parent: base
+ name: project2-test
+ nodeset: project2-nodeset
+
+ - project:
+ check:
+ jobs:
+ - project2-test
+ gate:
+ jobs:
+ - noop
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable', 'B',
+ files=file_dict)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 1, "B should report success")
+ self.assertHistory([
+ dict(name='project2-test', result='SUCCESS', changes='2,1',
+ node='ubuntu-xenial'),
+ ])
+
+ def test_nodeset_branch_duplicate(self):
+ # Test that we can create a duplicate nodeset on a different
+ # branch of the same project -- i.e., that when we branch
+ # master to stable on a project with a nodeset, nothing
+ # changes.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+ self.waitUntilSettled()
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1,
+ "A should report success")
+ self.assertHistory([
+ dict(name='project1-test', result='SUCCESS', changes='1,1',
+ node='ubuntu-xenial'),
+ ])
+
+ def test_nodeset_branch_error_same_branch(self):
+ # Test that we are unable to define a nodeset twice on the same
+ # project-branch.
+ in_repo_conf = textwrap.dedent(
+ """
+ - nodeset:
+ name: project1-nodeset
+ nodes: []
+ - nodeset:
+ name: project1-nodeset
+ nodes: []
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('already defined', A.messages[0])
+
+ def test_nodeset_branch_error_same_project(self):
+ # Test that we are unable to create a nodeset which differs
+ # from another with the same name -- i.e., that if we have a
+ # duplicate nodeset on multiple branches of the same project,
+ # they must be identical.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - nodeset:
+ name: project1-nodeset
+ nodes: []
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('does not match existing definition in branch master',
+ A.messages[0])
+
+ def test_nodeset_branch_error_other_project(self):
+ # Test that we are unable to create a nodeset with the same
+ # name as another. We're never allowed to have a nodeset with
+ # the same name outside of a project.
+ in_repo_conf = textwrap.dedent(
+ """
+ - nodeset:
+ name: project1-nodeset
+ nodes: []
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('already defined in project org/project1',
+ A.messages[0])
+
+
+class TestSemaphoreBranches(ZuulTestCase):
+ tenant_config_file = 'config/semaphore-branches/main.yaml'
+
+ def test_semaphore_branch(self):
+ # Test that we can use a semaphore defined in another branch of
+ # the same project.
+ self.create_branch('org/project2', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project2', 'stable'))
+ self.waitUntilSettled()
+
+ with open(os.path.join(FIXTURE_DIR,
+ 'config/semaphore-branches/git/',
+ 'org_project2/zuul-semaphore.yaml')) as f:
+ config = f.read()
+
+ file_dict = {'zuul.yaml': config}
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ parent: base
+ name: project2-test
+ semaphore: project2-semaphore
+
+ - project:
+ check:
+ jobs:
+ - project2-test
+ gate:
+ jobs:
+ - noop
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable', 'B',
+ files=file_dict)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 1, "B should report success")
+ self.assertHistory([
+ dict(name='project2-test', result='SUCCESS', changes='2,1')
+ ])
+
+ def test_semaphore_branch_duplicate(self):
+ # Test that we can create a duplicate semaphore on a different
+ # branch of the same project -- i.e., that when we branch
+ # master to stable on a project with a semaphore, nothing
+ # changes.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+ self.waitUntilSettled()
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1,
+ "A should report success")
+ self.assertHistory([
+ dict(name='project1-test', result='SUCCESS', changes='1,1')
+ ])
+
+ def test_semaphore_branch_error_same_branch(self):
+ # Test that we are unable to define a semaphore twice on the same
+ # project-branch.
+ in_repo_conf = textwrap.dedent(
+ """
+ - semaphore:
+ name: project1-semaphore
+ max: 2
+ - semaphore:
+ name: project1-semaphore
+ max: 2
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('already defined', A.messages[0])
+
+ def test_semaphore_branch_error_same_project(self):
+ # Test that we are unable to create a semaphore which differs
+ # from another with the same name -- i.e., that if we have a
+ # duplicate semaphore on multiple branches of the same project,
+ # they must be identical.
+ self.create_branch('org/project1', 'stable')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'org/project1', 'stable'))
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - semaphore:
+ name: project1-semaphore
+ max: 4
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('does not match existing definition in branch master',
+ A.messages[0])
+
+ def test_semaphore_branch_error_other_project(self):
+ # Test that we are unable to create a semaphore with the same
+ # name as another. We're never allowed to have a semaphore with
+ # the same name outside of a project.
+ in_repo_conf = textwrap.dedent(
+ """
+ - semaphore:
+ name: project1-semaphore
+ max: 2
+ """)
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('already defined in project org/project1',
+ A.messages[0])
+
+
class TestJobOutput(AnsibleZuulTestCase):
tenant_config_file = 'config/job-output/main.yaml'
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index bf11c6f..b299219 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -171,14 +171,6 @@
return pid_fn
@abc.abstractmethod
- def exit_handler(self, signum, frame):
- """
- This is a signal handler which is called on SIGINT and SIGTERM and must
- take care of stopping the application.
- """
- pass
-
- @abc.abstractmethod
def run(self):
"""
This is the main run method of the application.
@@ -197,8 +189,6 @@
signal.signal(signal.SIGUSR2, stack_dump_handler)
if self.args.nodaemon:
- signal.signal(signal.SIGTERM, self.exit_handler)
- signal.signal(signal.SIGINT, self.exit_handler)
self.run()
else:
# Exercise the pidfile before we do anything else (including
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 5b06f0c..b050a59 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -17,6 +17,7 @@
import logging
import os
import sys
+import signal
import tempfile
import zuul.cmd
@@ -50,6 +51,8 @@
def exit_handler(self, signum, frame):
self.executor.stop()
+ self.executor.join()
+ sys.exit(0)
def start_log_streamer(self):
pipe_read, pipe_write = os.pipe()
@@ -106,7 +109,16 @@
log_streaming_port=self.finger_port)
self.executor.start()
- self.executor.join()
+ if self.args.nodaemon:
+ signal.signal(signal.SIGTERM, self.exit_handler)
+ while True:
+ try:
+ signal.pause()
+ except KeyboardInterrupt:
+ print("Ctrl + C: asking executor to exit nicely...\n")
+ self.exit_handler(signal.SIGINT, None)
+ else:
+ self.executor.join()
def main():
diff --git a/zuul/cmd/fingergw.py b/zuul/cmd/fingergw.py
index 0d47f08..920eed8 100644
--- a/zuul/cmd/fingergw.py
+++ b/zuul/cmd/fingergw.py
@@ -14,6 +14,7 @@
# under the License.
import logging
+import signal
import sys
import zuul.cmd
@@ -46,9 +47,6 @@
if self.args.command:
self.args.nodaemon = True
- def exit_handler(self, signum, frame):
- self.stop()
-
def run(self):
'''
Main entry point for the FingerGatewayApp.
@@ -86,7 +84,19 @@
self.log.info('Starting Zuul finger gateway app')
self.gateway.start()
- self.gateway.wait()
+ if self.args.nodaemon:
+ # NOTE(Shrews): When running in non-daemon mode, although sending
+ # the 'stop' command via the command socket will shutdown the
+ # gateway, it's still necessary to Ctrl+C to stop the app.
+ while True:
+ try:
+ signal.pause()
+ except KeyboardInterrupt:
+ print("Ctrl + C: asking gateway to exit nicely...\n")
+ self.stop()
+ break
+ else:
+ self.gateway.wait()
self.log.info('Stopped Zuul finger gateway app')
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
index 390191f..8c47989 100755
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -14,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import signal
import sys
import zuul.cmd
@@ -43,6 +44,8 @@
def exit_handler(self, signum, frame):
self.merger.stop()
+ self.merger.join()
+ sys.exit(0)
def run(self):
# See comment at top of file about zuul imports
@@ -59,7 +62,16 @@
self.connections)
self.merger.start()
- self.merger.join()
+ if self.args.nodaemon:
+ signal.signal(signal.SIGTERM, self.exit_handler)
+ while True:
+ try:
+ signal.pause()
+ except KeyboardInterrupt:
+ print("Ctrl + C: asking merger to exit nicely...\n")
+ self.exit_handler(signal.SIGINT, None)
+ else:
+ self.merger.join()
def main():
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index 3cffa10..7748a80 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -63,7 +63,9 @@
def exit_handler(self, signum, frame):
self.sched.exit()
+ self.sched.join()
self.stop_gear_server()
+ sys.exit(0)
def start_gear_server(self):
pipe_read, pipe_write = os.pipe()
@@ -173,7 +175,16 @@
signal.signal(signal.SIGHUP, self.reconfigure_handler)
- self.sched.join()
+ if self.args.nodaemon:
+ signal.signal(signal.SIGTERM, self.exit_handler)
+ while True:
+ try:
+ signal.pause()
+ except KeyboardInterrupt:
+ print("Ctrl + C: asking scheduler to exit nicely...\n")
+ self.exit_handler(signal.SIGINT, None)
+ else:
+ self.sched.join()
def main():
diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py
index 78392db..ad3062f 100755
--- a/zuul/cmd/web.py
+++ b/zuul/cmd/web.py
@@ -89,6 +89,12 @@
name='web')
self.thread.start()
+ try:
+ signal.pause()
+ except KeyboardInterrupt:
+ print("Ctrl + C: asking web server to exit nicely...\n")
+ self.exit_handler(signal.SIGINT, None)
+
self.thread.join()
loop.stop()
loop.close()
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d622370..4f93907 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -407,7 +407,7 @@
@staticmethod
def fromYaml(conf, anonymous=False):
NodeSetParser.getSchema(anonymous)(conf)
- ns = model.NodeSet(conf.get('name'))
+ ns = model.NodeSet(conf.get('name'), conf.get('_source_context'))
node_names = set()
group_names = set()
for conf_node in as_list(conf['nodes']):
@@ -1597,7 +1597,8 @@
classes = TenantParser._getLoadClasses(tenant, config_secret)
if 'secret' not in classes:
continue
- layout.addSecret(SecretParser.fromYaml(layout, config_secret))
+ with configuration_exceptions('secret', config_secret):
+ layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in data.jobs:
classes = TenantParser._getLoadClasses(tenant, config_job)
@@ -1621,23 +1622,22 @@
if parent:
layout.getJob(parent)
- if not skip_semaphores:
- for config_semaphore in data.semaphores:
- classes = TenantParser._getLoadClasses(
- tenant, config_semaphore)
- if 'semaphore' not in classes:
- continue
+ if skip_semaphores:
+ # We should not actually update the layout with new
+ # semaphores, but so that we can validate that the config
+ # is correct, create a shadow layout here to which we add
+ # new semaphores so validation is complete.
+ semaphore_layout = model.Layout(tenant)
+ else:
+ semaphore_layout = layout
+ for config_semaphore in data.semaphores:
+ classes = TenantParser._getLoadClasses(
+ tenant, config_semaphore)
+ if 'semaphore' not in classes:
+ continue
+ with configuration_exceptions('semaphore', config_semaphore):
semaphore = SemaphoreParser.fromYaml(config_semaphore)
- old_semaphore = layout.semaphores.get(semaphore.name)
- if (old_semaphore and
- (old_semaphore.source_context.project ==
- semaphore.source_context.project)):
- # If a semaphore shows up twice in the same
- # project, it's probably due to showing up in
- # two branches. Ignore subsequent
- # definitions.
- continue
- layout.addSemaphore(semaphore)
+ semaphore_layout.addSemaphore(semaphore)
project_template_parser = ProjectTemplateParser(tenant, layout)
for config_template in data.project_templates:
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 33c66f9..3b3f1ae 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -170,6 +170,16 @@
connection = self.connections[connection_name]
return connection.driver.getTrigger(connection, config)
+ def getSourceByHostname(self, hostname):
+ for connection in self.connections.values():
+ if hasattr(connection, 'canonical_hostname'):
+ if connection.canonical_hostname == hostname:
+ return self.getSource(connection.connection_name)
+ if hasattr(connection, 'server'):
+ if connection.server == hostname:
+ return self.getSource(connection.connection_name)
+ return None
+
def getSourceByCanonicalHostname(self, canonical_hostname):
for connection in self.connections.values():
if hasattr(connection, 'canonical_hostname'):
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index b8a280f..88ddf7d 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -360,7 +360,7 @@
url = urllib.parse.urlparse(match)
except ValueError:
continue
- source = self.sched.connections.getSourceByCanonicalHostname(
+ source = self.sched.connections.getSourceByHostname(
url.hostname)
if not source:
continue
@@ -722,9 +722,10 @@
return True
def onBuildCompleted(self, build):
- self.log.debug("Build %s completed" % build)
item = build.build_set.item
+ self.log.debug("Build %s of %s completed" % (build, item.change))
+
item.setResult(build)
item.pipeline.layout.tenant.semaphore_handler.release(item, build.job)
self.log.debug("Item %s status is now:\n %s" %
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index bd4ca58..035dbf5 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -143,19 +143,30 @@
self.update()
repo = self.createRepoObject()
origin = repo.remotes.origin
+ seen = set()
+ head = None
+ stale_refs = origin.stale_refs
+ # Update our local heads to match the remote, and pick one to
+ # reset the repo to. We don't delete anything at this point
+ # because we want to make sure the repo is in a state stable
+ # enough for git to operate.
for ref in origin.refs:
if ref.remote_head == 'HEAD':
continue
+ if ref in stale_refs:
+ continue
repo.create_head(ref.remote_head, ref, force=True)
-
- # try reset to remote HEAD (usually origin/master)
- # If it fails, pick the first reference
- try:
- repo.head.reference = origin.refs['HEAD']
- except IndexError:
- repo.head.reference = origin.refs[0]
+ seen.add(ref.remote_head)
+ if head is None:
+ head = ref.remote_head
+ self.log.debug("Reset to %s", head)
+ repo.head.reference = head
reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
+ for ref in stale_refs:
+ self.log.debug("Delete stale ref %s", ref.remote_head)
+ repo.delete_head(ref.remote_head, force=True)
+ git.refs.RemoteReference.delete(repo, ref, force=True)
def prune(self):
repo = self.createRepoObject()
@@ -163,7 +174,7 @@
stale_refs = origin.stale_refs
if stale_refs:
self.log.debug("Pruning stale refs: %s", stale_refs)
- git.refs.RemoteReference.delete(repo, *stale_refs)
+ git.refs.RemoteReference.delete(repo, force=True, *stale_refs)
def getBranchHead(self, branch):
repo = self.createRepoObject()
@@ -193,11 +204,12 @@
return repo.refs
def setRef(self, path, hexsha, repo=None):
+ self.log.debug("Create reference %s at %s in %s",
+ path, hexsha, self.local_path)
if repo is None:
repo = self.createRepoObject()
binsha = gitdb.util.to_bin_sha(hexsha)
obj = git.objects.Object.new_from_sha(repo, binsha)
- self.log.debug("Create reference %s", path)
git.refs.Reference.create(repo, path, obj, force=True)
def setRefs(self, refs):
diff --git a/zuul/model.py b/zuul/model.py
index 29c5a9d..96ec85b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -476,8 +476,9 @@
or they may appears anonymously in in-line job definitions.
"""
- def __init__(self, name=None):
+ def __init__(self, name=None, source_context=None):
self.name = name or ''
+ self.source_context = source_context
self.nodes = OrderedDict()
self.groups = OrderedDict()
@@ -612,6 +613,9 @@
self.source_context == other.source_context and
self.secret_data == other.secret_data)
+ def areDataEqual(self, other):
+ return (self.secret_data == other.secret_data)
+
def __repr__(self):
return '<Secret %s>' % (self.name,)
@@ -662,7 +666,6 @@
if not isinstance(other, SourceContext):
return False
return (self.project == other.project and
- self.branch == other.branch and
self.trusted == other.trusted)
def __ne__(self, other):
@@ -2584,18 +2587,65 @@
return True
def addNodeSet(self, nodeset):
- if nodeset.name in self.nodesets:
- raise Exception("NodeSet %s already defined" % (nodeset.name,))
+ # It's ok to have a duplicate nodeset definition, but only if
+ # they are in different branches of the same repo, and have
+ # the same values.
+ other = self.nodesets.get(nodeset.name)
+ if other is not None:
+ if not nodeset.source_context.isSameProject(other.source_context):
+ raise Exception("Nodeset %s already defined in project %s" %
+ (nodeset.name, other.source_context.project))
+ if nodeset.source_context.branch == other.source_context.branch:
+ raise Exception("Nodeset %s already defined" % (nodeset.name,))
+ if nodeset != other:
+ raise Exception("Nodeset %s does not match existing definition"
+ " in branch %s" %
+ (nodeset.name, other.source_context.branch))
+ # Identical data in a different branch of the same project;
+ # ignore the duplicate definition
+ return
self.nodesets[nodeset.name] = nodeset
def addSecret(self, secret):
- if secret.name in self.secrets:
- raise Exception("Secret %s already defined" % (secret.name,))
+ # It's ok to have a duplicate secret definition, but only if
+ # they are in different branches of the same repo, and have
+ # the same values.
+ other = self.secrets.get(secret.name)
+ if other is not None:
+ if not secret.source_context.isSameProject(other.source_context):
+ raise Exception("Secret %s already defined in project %s" %
+ (secret.name, other.source_context.project))
+ if secret.source_context.branch == other.source_context.branch:
+ raise Exception("Secret %s already defined" % (secret.name,))
+ if not secret.areDataEqual(other):
+ raise Exception("Secret %s does not match existing definition"
+ " in branch %s" %
+ (secret.name, other.source_context.branch))
+ # Identical data in a different branch of the same project;
+ # ignore the duplicate definition
+ return
self.secrets[secret.name] = secret
def addSemaphore(self, semaphore):
- if semaphore.name in self.semaphores:
- raise Exception("Semaphore %s already defined" % (semaphore.name,))
+ # It's ok to have a duplicate semaphore definition, but only if
+ # they are in different branches of the same repo, and have
+ # the same values.
+ other = self.semaphores.get(semaphore.name)
+ if other is not None:
+ if not semaphore.source_context.isSameProject(
+ other.source_context):
+ raise Exception("Semaphore %s already defined in project %s" %
+ (semaphore.name, other.source_context.project))
+ if semaphore.source_context.branch == other.source_context.branch:
+ raise Exception("Semaphore %s already defined" %
+ (semaphore.name,))
+ if semaphore != other:
+ raise Exception("Semaphore %s does not match existing"
+ " definition in branch %s" %
+ (semaphore.name, other.source_context.branch))
+ # Identical data in a different branch of the same project;
+ # ignore the duplicate definition
+ return
self.semaphores[semaphore.name] = semaphore
def addPipeline(self, pipeline):
@@ -2750,6 +2800,15 @@
self.name = name
self.max = int(max)
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, Semaphore):
+ return False
+ return (self.name == other.name and
+ self.max == other.max)
+
class SemaphoreHandler(object):
log = logging.getLogger("zuul.SemaphoreHandler")