Merge "Simplify the log url" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index 98b880d..50223fa 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -31,7 +31,7 @@
- job:
name: tox-linters
parent: tox
- run: tox/docs
+ run: tox/linters
- job:
name: tox-py27
diff --git a/README.rst b/README.rst
index 932edbf..c55f7b3 100644
--- a/README.rst
+++ b/README.rst
@@ -58,7 +58,7 @@
Some of the information in the specs may be effectively superceded
by changes here, which are still undergoing review.
-4) Read documentation on the internal data model and testing: http://docs.openstack.org/infra/zuul/feature/zuulv3/internals.html
+4) Read developer documentation on the internal data model and testing: http://docs.openstack.org/infra/zuul/feature/zuulv3/developer.html
The general philosophy for Zuul tests is to perform functional
testing of either the individual component or the entire end-to-end
diff --git a/bindep.txt b/bindep.txt
index 8d8c45b..6895444 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -4,4 +4,13 @@
mysql-client [test]
mysql-server [test]
libjpeg-dev [test]
+openssl [test]
zookeeperd [platform:dpkg]
+build-essential [platform:dpkg]
+gcc [platform:rpm]
+libssl-dev [platform:dpkg]
+openssl-devel [platform:rpm]
+libffi-dev [platform:dpkg]
+libffi-devel [platform:rpm]
+python-dev [platform:dpkg]
+python-devel [platform:rpm]
diff --git a/doc/source/datamodel.rst b/doc/source/developer/datamodel.rst
similarity index 100%
rename from doc/source/datamodel.rst
rename to doc/source/developer/datamodel.rst
diff --git a/doc/source/drivers.rst b/doc/source/developer/drivers.rst
similarity index 100%
rename from doc/source/drivers.rst
rename to doc/source/developer/drivers.rst
diff --git a/doc/source/developer.rst b/doc/source/developer/index.rst
similarity index 95%
rename from doc/source/developer.rst
rename to doc/source/developer/index.rst
index 527ea6e..986bbe4 100644
--- a/doc/source/developer.rst
+++ b/doc/source/developer/index.rst
@@ -12,4 +12,5 @@
datamodel
drivers
+ triggers
testing
diff --git a/doc/source/testing.rst b/doc/source/developer/testing.rst
similarity index 100%
rename from doc/source/testing.rst
rename to doc/source/developer/testing.rst
diff --git a/doc/source/developer/triggers.rst b/doc/source/developer/triggers.rst
new file mode 100644
index 0000000..56f4a03
--- /dev/null
+++ b/doc/source/developer/triggers.rst
@@ -0,0 +1,19 @@
+Triggers
+========
+
+Triggers must inherit from :py:class:`~zuul.trigger.BaseTrigger` and, at a minimum,
+implement the :py:meth:`~zuul.trigger.BaseTrigger.getEventFilters` method.
+
+.. autoclass:: zuul.trigger.BaseTrigger
+ :members:
+
+Current list of triggers are:
+
+.. autoclass:: zuul.driver.gerrit.gerrittrigger.GerritTrigger
+ :members:
+
+.. autoclass:: zuul.driver.timer.timertrigger.TimerTrigger
+ :members:
+
+.. autoclass:: zuul.driver.zuul.zuultrigger.ZuulTrigger
+ :members:
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 3f903db..fb30b92 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -24,7 +24,7 @@
executors
statsd
client
- developer
+ developer/index
Indices and tables
==================
diff --git a/playbooks/roles/prepare-workspace/tasks/main.yaml b/playbooks/roles/prepare-workspace/tasks/main.yaml
index c5952c7..4d42b2d 100644
--- a/playbooks/roles/prepare-workspace/tasks/main.yaml
+++ b/playbooks/roles/prepare-workspace/tasks/main.yaml
@@ -19,3 +19,4 @@
synchronize:
dest: "{{ zuul_workspace_root }}"
src: "{{ zuul.executor.src_root }}"
+ no_log: true
diff --git a/requirements.txt b/requirements.txt
index 186e7f6..c7e059a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,3 +19,4 @@
kazoo
sqlalchemy
alembic
+cryptography>=1.6
diff --git a/tests/base.py b/tests/base.py
index 9a6fb69..2816b9f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -50,6 +50,7 @@
import testtools.content
import testtools.content_type
from git.exc import NoSuchPathError
+import yaml
import zuul.driver.gerrit.gerritsource as gerritsource
import zuul.driver.gerrit.gerritconnection as gerritconnection
@@ -1213,6 +1214,11 @@
different tenant/project layout while using the standard main
configuration.
+ :cvar bool create_project_keys: Indicates whether Zuul should
+ auto-generate keys for each project, or whether the test
+ infrastructure should insert dummy keys to save time during
+ startup. Defaults to False.
+
The following are instance variables that are useful within test
methods:
@@ -1244,6 +1250,7 @@
config_file = 'zuul.conf'
run_ansible = False
+ create_project_keys = False
def _startMerger(self):
self.merge_server = zuul.merger.server.MergeServer(self.config,
@@ -1438,6 +1445,39 @@
project = reponame.replace('_', '/')
self.copyDirToRepo(project,
os.path.join(git_path, reponame))
+ self.setupAllProjectKeys()
+
+ def setupAllProjectKeys(self):
+ if self.create_project_keys:
+ return
+
+ path = self.config.get('zuul', 'tenant_config')
+ with open(os.path.join(FIXTURE_DIR, path)) as f:
+ tenant_config = yaml.safe_load(f.read())
+ for tenant in tenant_config:
+ sources = tenant['tenant']['source']
+ for source, conf in sources.items():
+ for project in conf.get('config-repos', []):
+ self.setupProjectKeys(source, project)
+ for project in conf.get('project-repos', []):
+ self.setupProjectKeys(source, project)
+
+ def setupProjectKeys(self, source, project):
+ # Make sure we set up an RSA key for the project so that we
+ # don't spend time generating one:
+
+ key_root = os.path.join(self.state_root, 'keys')
+ if not os.path.isdir(key_root):
+ os.mkdir(key_root, 0o700)
+ private_key_file = os.path.join(key_root, source, project + '.pem')
+ private_key_dir = os.path.dirname(private_key_file)
+ self.log.debug("Installing test keys for project %s at %s" % (
+ project, private_key_file))
+ if not os.path.isdir(private_key_dir):
+ os.makedirs(private_key_dir)
+ with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
+ with open(private_key_file, 'w') as o:
+ o.write(i.read())
def setupZK(self):
self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
@@ -1473,6 +1513,22 @@
self.assertFalse(node['_lock'], "Node %s is locked" %
(node['_oid'],))
+ def assertNoGeneratedKeys(self):
+ # Make sure that Zuul did not generate any project keys
+ # (unless it was supposed to).
+
+ if self.create_project_keys:
+ return
+
+ with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
+ test_key = i.read()
+
+ key_root = os.path.join(self.state_root, 'keys')
+ for root, dirname, files in os.walk(key_root):
+ for fn in files:
+ with open(os.path.join(root, fn)) as f:
+ self.assertEqual(test_key, f.read())
+
def assertFinalState(self):
# Make sure that git.Repo objects have been garbage collected.
repos = []
@@ -1484,6 +1540,7 @@
self.assertEqual(len(repos), 0)
self.assertEmptyQueues()
self.assertNodepoolState()
+ self.assertNoGeneratedKeys()
ipm = zuul.manager.independent.IndependentPipelineManager
for tenant in self.sched.abide.tenants.values():
for pipeline in tenant.layout.pipelines.values():
@@ -1850,6 +1907,7 @@
f.close()
self.config.set('zuul', 'tenant_config',
os.path.join(FIXTURE_DIR, f.name))
+ self.setupAllProjectKeys()
def addCommitToRepo(self, project, message, files,
branch='master', tag=None):
@@ -1878,11 +1936,18 @@
def commitLayoutUpdate(self, orig_name, source_name):
source_path = os.path.join(self.test_root, 'upstream',
- source_name, 'zuul.yaml')
- with open(source_path, 'r') as nt:
- before = self.addCommitToRepo(
- orig_name, 'Pulling content from %s' % source_name,
- {'zuul.yaml': nt.read()})
+ source_name)
+ to_copy = ['zuul.yaml']
+ for playbook in os.listdir(os.path.join(source_path, 'playbooks')):
+ to_copy.append('playbooks/{}'.format(playbook))
+ commit_data = {}
+ for source_file in to_copy:
+ source_file_path = os.path.join(source_path, source_file)
+ with open(source_file_path, 'r') as nt:
+ commit_data[source_file] = nt.read()
+ before = self.addCommitToRepo(
+ orig_name, 'Pulling content from %s' % source_name,
+ commit_data)
return before
def addEvent(self, connection, event):
diff --git a/tests/encrypt_secret.py b/tests/encrypt_secret.py
new file mode 100644
index 0000000..b8524a0
--- /dev/null
+++ b/tests/encrypt_secret.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+import os
+
+from zuul.lib import encryption
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+ 'fixtures')
+
+
+def main():
+ private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
+ with open(private_key_file, "rb") as f:
+ private_key, public_key = \
+ encryption.deserialize_rsa_keypair(f.read())
+
+ ciphertext = encryption.encrypt_pkcs1_oaep(sys.argv[1], public_key)
+ print(ciphertext.encode('base64'))
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
index 45acb87..3371a20 100644
--- a/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
@@ -6,5 +6,8 @@
- copy:
src: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
dest: "{{zuul._test.test_root}}/{{zuul.uuid}}.copied"
+ - copy:
+ content: "{{test_secret.username}} {{test_secret.password}}"
+ dest: "{{zuul._test.test_root}}/{{zuul.uuid}}.secrets"
roles:
- bare-role
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index 5c6c998..0980bc1 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -2,6 +2,7 @@
name: check
manager: independent
source: gerrit
+ allow-secrets: true
trigger:
gerrit:
- event: patchset-created
@@ -34,6 +35,21 @@
verified: 0
precedence: high
+- secret:
+ name: test_secret
+ data:
+ username: test-username
+ password: !encrypted/pkcs1-oaep |
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+ ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+ 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+ Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+ xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+ aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+ +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
- job:
name: python27
pre-run: pre
@@ -42,6 +58,9 @@
flagpath: '{{zuul._test.test_root}}/{{zuul.uuid}}.flag'
roles:
- zuul: bare-role
+ auth:
+ secrets:
+ - test_secret
- job:
parent: python27
diff --git a/tests/fixtures/config/single-tenant/git/layout-no-jobs/playbooks/gate-noop.yaml b/tests/fixtures/config/single-tenant/git/layout-no-jobs/playbooks/gate-noop.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-no-jobs/playbooks/gate-noop.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-no-jobs/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-no-jobs/zuul.yaml
new file mode 100644
index 0000000..5894440
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-no-jobs/zuul.yaml
@@ -0,0 +1,49 @@
+- pipeline:
+ name: check
+ manager: independent
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: gate-noop
+
+- project:
+ name: org/project
+ merge-mode: cherry-pick
+ check:
+ jobs:
+ - gate-noop
+ gate:
+ jobs:
+ - gate-noop
diff --git a/tests/fixtures/config/single-tenant/main.yaml b/tests/fixtures/config/single-tenant/main.yaml
index a22ed5c..d9868fa 100644
--- a/tests/fixtures/config/single-tenant/main.yaml
+++ b/tests/fixtures/config/single-tenant/main.yaml
@@ -4,3 +4,5 @@
gerrit:
config-repos:
- common-config
+ project-repos:
+ - org/project
diff --git a/tests/fixtures/layout-no-jobs.yaml b/tests/fixtures/layout-no-jobs.yaml
deleted file mode 100644
index e860ad5..0000000
--- a/tests/fixtures/layout-no-jobs.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-includes:
- - python-file: custom_functions.py
-
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- start:
- gerrit:
- verified: 0
- precedence: high
-
-projects:
- - name: org/project
- merge-mode: cherry-pick
- check:
- - gate-noop
- gate:
- - gate-noop
diff --git a/tests/fixtures/private.pem b/tests/fixtures/private.pem
new file mode 100644
index 0000000..fa709b6
--- /dev/null
+++ b/tests/fixtures/private.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEAsGqZLUUwV/EZJKddMS206mH7qYmqYhWLo/TUlpDt2JuEaBqC
+YV8mF9LsjpoqM/Pp0U/r5aQLDUXbRLDn+K+NqbvTJajYxHJicP1CAWg1eKUNZjUa
+ya5HP4Ow1hS7AeiF4TSRdiwtHT/gJO2NSsavyc30/meKt0WBgbYlrBB81HEQjYWn
+ajf/4so5E8DdrC9tAqmmzde1qcTz7ULouIz53hjp/U3yVMFbpawv194jzHvddmAX
+3aEUByx2t6lP7dhOAEIEmzmh15hRbacxQI5aYWv+ZR0z9PqdwwD+DBbb1AwiX5MJ
+jtIoVCmkEZvcUFiDicyteNMCa5ulpj2SF0oH4MlialOP6MiJnmxklDYO07AM/qom
+cU55pCD8ctu1yD/UydecLk0Uj/9XxqmPQJFEcstdXJZQfr5ZNnChOEg6oQ9UImWj
+av8HQsA6mFW1oAKbDMrgEewooWriqGW5pYtR7JBfph6Mt5HGaeH4uqYpb1fveHG1
+ODa7HBnlNo3qMendBb2wzHGCgtUgWnGfp24TsUOUlndCXYhsYbOZbCTW5GwElK0G
+ri06KPpybY43AIaxcxqilVh5Eapmq7axBm4ZzbTOfv15L0FIemEGgpnklevQbZNL
+IrcE0cS/13qJUvFaYX4yjrtEnzZ3ntjXrpFdgLPBKn7Aqf6lWz6BPi07axECAwEA
+AQKCAgEAkoPltYhZ7x+ojx2Es1xPfb1kwlg4Ln/QWpnymR3Cu3vlioRBtlbMj0q4
+9nIpDL7NeO4Ub8M+/oX+5ly6O3qpf8cjRIqnhPeutEJRuFNw3ULPDwyZs9hPCfv4
+OMQ80AfqcLA1At0Lltg+8sxr5SeARW0MxOD/fth2B2FchjunQNSqN69B7GCX3yWu
+I66xK9izg1uc0iYNlPKi13ETUHqc5ozwgFRlJ2jzEXQgw/qU5rYUpsSF7aZiuNZ/
+vmcan+FeXq51nulNdX3mWthZelD/1RtYy2dmiFZAAf1oAGhXqBNv1MqMTJZTshpn
+TcyRPBVXIXHgvJEa2H4LJDbMhxUP1opJ+Vxa8Cy6I60O8TwPBHwL83K5oH4yugun
+AP2hWZxFMK9YcVliJwt3Mjozuh5vCRF9+7oqi0fASuhOY+eYNQAtcPK9WBti6qmN
+hUO4bdx+r+UEb8TliUDH+x5lNmKc2pgptYS+O8+oB2vh2V7e0mwvc3jg4S7E5Ukm
+y4Y9JS0c4q352W0lrfPCDYwzXEpK8mmCjvBC/w320Yi2HJwqkfYQThgEbzOP37dW
+Ei+0+cu6RuA4H+1DozkrWybFw6Ju12IE4vfbliyht1yuj0+/Rpevp1KpFKuy5xSB
+1Jq3lGxTFDGle7nRBc2JwfIu63texnmvTwKlx1+w0tqpY/gVZhUCggEBAOAzVHum
+luqKVewWT8yR4mZx4jiWdxLch3Q+scMq2mthQ5773Of0P2r45iJz7jDS7fT0yuRF
+gBpqygX42xe+wqJleKAzKyMQ9aWtYRszfCz6Ob9kLTtoi0/Xuo5dMyg41BRHAatr
+acj9NXBEvRS4oNKw3nxEVayBjvYN5LwLAzGNorXCkt9E+72eWJU6eg0CQQxwI2rG
+f/S+niMtLDWfayHPu7KBKRVlUu1kI07JF1eSJmsHBcTN1+CaXuN82Ty+ucdtjRWR
+5FyLZxaceLGrY5so87pH7kcBB2+H7ovuash7g+CT3XyDcQACWTjTszIpt6fGO6ux
+7Tea5/OOLaJiaI8CggEBAMlwPPW3HQzC6dqwBVNgVYQh9ZEoygKOWNMPNE1TuqUU
+boJLazQI5Qd/qm17otAnDrIX7cEB/+6xiQPZkw6lsqdzGHNBSXb8OPYvLDBHq2oR
+oNjdW4/c5znBL3ExXqEJIHAl9FWc5YLRvboHwtkKCpK5mdlZyoMVsBX62IFodAhK
+a8oQiLvYjOwFOay3sOMdhc+ndupw7b9MaAsbe1w7DW3Y7I/bHstxiriDfuTI/nt7
+MPZBzj9afqWHEJ3TWwuJ1IuUhHupf9ylA06GfBgerWSlp90yVfbZNQDljtdNwIZW
+oBLF6EhZxh6ka8iodeS4cduxEV3BoofMXjIjVReCgl8CggEBALSwabwl7Kclyk21
+RabnRAGwctOMYHbxCLHk/Tr/xHyaLPdqoQTH0nySEFdf+22Z8XFkAEiswquHuT3K
+7Dhc41wiT289Ddz7BB78drCHc+KD4Bqhz9p7TRuSD6ZA8sPN2Q5mk6/lp6H2gCT1
+ITYb/nEPXp/kKvAWknM3i0sJzQ8YyTOXluseG40cmuPZ9xeY43f0wHaDeAh1v9k1
+xNWKn7rmQq2Abu3xdT4hYFtUsd0/ynqjdEDCbON1Rlgs/J96Txus7PGfXN5A81pD
+zPnT2TjpblSJOD49VBLNCLH5+lGNSiGqyexZuq55NhMYeulIud0bZGfhw/72d03R
+HnIqwX0CggEBAKiKglbMuT+eLfBN6obSSXretwqXaD4vP96IECjK75WDvNrDo5TM
+BGT7ymsEUTt8Em2sW79rnunmHU/dUY+l0A8O29xDOeaWLkq9OWnD7YY37a7FtwBt
+wgGuw7Ufq59tdXigKQkg119XgjkOmVbjcelF5ZXX7Ps0wDoDwfa0oLD3I6zTnLQf
+AfnQfWsn3paIcxdFdNe/WQ0ALuVsPxDyT9Ai+ft7SQ7Ll1e+ngNqsJI8hsDkWl7j
+pqd0lNCYsMq8rduDjj2xmkvQvS2MlHPR5x4ZBJSsswRwxEpVx+gZJAbCn/hVIn62
+rm+g/pXLbajLMmiwhGk/xG9+7SliKqYbCl0CggEATQtwqAVPdwzT5XaRS1CeLId5
+sZD8mP5WLBKas69nfISilcUKqJjqTTqxfXs60wOK3/r43B+7QLitfPLRqf0hRQT9
+6HQG1YGx1FfZwgsP5SJKpAGGjenhsSTwpMJJI5s2I2e1O01frF2qEodqmRUwHXbh
+rGXqzAHLieaBzHjSvS2Z4kGVu6ZbpRXSNTSiiF+z8O9PCahzNFrC/ty+lbtxcqhf
+wHttEccW1TmiuB9GD23NI96zLsjZALvdqpvHMf5OHiDdLmI+Ap7qlR04V3bDDzF4
+B6HR6bRxVZQQWaEwE1RfuDgj5Msrbcgq0yFayPvXGiIIrAUWkUUQVsUU/TOfBQ==
+-----END RSA PRIVATE KEY-----
diff --git a/tests/fixtures/public.pem b/tests/fixtures/public.pem
new file mode 100644
index 0000000..33a78c4
--- /dev/null
+++ b/tests/fixtures/public.pem
@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsGqZLUUwV/EZJKddMS20
+6mH7qYmqYhWLo/TUlpDt2JuEaBqCYV8mF9LsjpoqM/Pp0U/r5aQLDUXbRLDn+K+N
+qbvTJajYxHJicP1CAWg1eKUNZjUaya5HP4Ow1hS7AeiF4TSRdiwtHT/gJO2NSsav
+yc30/meKt0WBgbYlrBB81HEQjYWnajf/4so5E8DdrC9tAqmmzde1qcTz7ULouIz5
+3hjp/U3yVMFbpawv194jzHvddmAX3aEUByx2t6lP7dhOAEIEmzmh15hRbacxQI5a
+YWv+ZR0z9PqdwwD+DBbb1AwiX5MJjtIoVCmkEZvcUFiDicyteNMCa5ulpj2SF0oH
+4MlialOP6MiJnmxklDYO07AM/qomcU55pCD8ctu1yD/UydecLk0Uj/9XxqmPQJFE
+cstdXJZQfr5ZNnChOEg6oQ9UImWjav8HQsA6mFW1oAKbDMrgEewooWriqGW5pYtR
+7JBfph6Mt5HGaeH4uqYpb1fveHG1ODa7HBnlNo3qMendBb2wzHGCgtUgWnGfp24T
+sUOUlndCXYhsYbOZbCTW5GwElK0Gri06KPpybY43AIaxcxqilVh5Eapmq7axBm4Z
+zbTOfv15L0FIemEGgpnklevQbZNLIrcE0cS/13qJUvFaYX4yjrtEnzZ3ntjXrpFd
+gLPBKn7Aqf6lWz6BPi07axECAwEAAQ==
+-----END PUBLIC KEY-----
diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py
new file mode 100644
index 0000000..4dda78b
--- /dev/null
+++ b/tests/unit/test_encryption.py
@@ -0,0 +1,69 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import subprocess
+import tempfile
+
+from zuul.lib import encryption
+
+from tests.base import BaseTestCase
+
+
+class TestEncryption(BaseTestCase):
+
+ def setUp(self):
+ super(TestEncryption, self).setUp()
+ self.private, self.public = encryption.generate_rsa_keypair()
+
+ def test_serialization(self):
+ "Verify key serialization"
+ pem_private = encryption.serialize_rsa_private_key(self.private)
+ private2, public2 = encryption.deserialize_rsa_keypair(pem_private)
+
+ # cryptography public / private key objects don't implement
+ # equality testing, so we make sure they have the same numbers.
+ self.assertEqual(self.private.private_numbers(),
+ private2.private_numbers())
+ self.assertEqual(self.public.public_numbers(),
+ public2.public_numbers())
+
+ def test_pkcs1_oaep(self):
+ "Verify encryption and decryption"
+ orig_plaintext = "some text to encrypt"
+ ciphertext = encryption.encrypt_pkcs1_oaep(orig_plaintext, self.public)
+ plaintext = encryption.decrypt_pkcs1_oaep(ciphertext, self.private)
+ self.assertEqual(orig_plaintext, plaintext)
+
+ def test_openssl_pkcs1_oaep(self):
+ "Verify that we can decrypt something encrypted with OpenSSL"
+ orig_plaintext = "some text to encrypt"
+ pem_public = encryption.serialize_rsa_public_key(self.public)
+ public_file = tempfile.NamedTemporaryFile(delete=False)
+ try:
+ public_file.write(pem_public)
+ public_file.close()
+
+ p = subprocess.Popen(['openssl', 'rsautl', '-encrypt',
+ '-oaep', '-pubin', '-inkey',
+ public_file.name],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ (stdout, stderr) = p.communicate(orig_plaintext)
+ ciphertext = stdout
+ finally:
+ os.unlink(public_file.name)
+
+ plaintext = encryption.decrypt_pkcs1_oaep(ciphertext, self.private)
+ self.assertEqual(orig_plaintext, plaintext)
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index ee7c6ab..f906095 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -22,15 +22,33 @@
from zuul import model
from zuul import configloader
+from zuul.lib import encryption
-from tests.base import BaseTestCase
+from tests.base import BaseTestCase, FIXTURE_DIR
+
+
+class FakeSource(object):
+ def __init__(self, name):
+ self.name = name
class TestJob(BaseTestCase):
def setUp(self):
super(TestJob, self).setUp()
- self.project = model.Project('project', None)
+ self.tenant = model.Tenant('tenant')
+ self.layout = model.Layout()
+ self.project = model.Project('project', 'connection')
+ self.source = FakeSource('connection')
+ self.tenant.addProjectRepo(self.source, self.project)
+ self.pipeline = model.Pipeline('gate', self.layout)
+ self.layout.addPipeline(self.pipeline)
+ self.queue = model.ChangeQueue(self.pipeline)
+
+ private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
+ with open(private_key_file, "rb") as f:
+ self.project.private_key, self.project.public_key = \
+ encryption.deserialize_rsa_keypair(f.read())
self.context = model.SourceContext(self.project, 'master',
'test', True)
self.start_mark = yaml.Mark('name', 0, 0, 0, '', 0)
@@ -73,7 +91,7 @@
base.pre_run = [base_pre]
base.run = [base_run]
base.post_run = [base_post]
- base.auth = dict(foo='bar', inherit=False)
+ base.auth = model.AuthContext()
py27 = model.Job('py27')
self.assertEqual(None, py27.timeout)
@@ -85,7 +103,7 @@
[x.path for x in py27.run])
self.assertEqual(['base-post'],
[x.path for x in py27.post_run])
- self.assertEqual({}, py27.auth)
+ self.assertEqual(None, py27.auth)
def test_job_variants(self):
# This simulates freezing a job.
@@ -99,7 +117,8 @@
py27.pre_run = [py27_pre]
py27.run = [py27_run]
py27.post_run = [py27_post]
- auth = dict(foo='bar', inherit=False)
+ auth = model.AuthContext()
+ auth.secrets.append('foo')
py27.auth = auth
job = py27.copy()
@@ -302,6 +321,29 @@
tenant = model.Tenant('tenant')
layout = model.Layout()
+ conf = yaml.safe_load('''
+- secret:
+ name: pypi-credentials
+ data:
+ username: test-username
+ password: !encrypted/pkcs1-oaep |
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+ ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+ 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+ Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+ xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+ aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+ +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+''')[0]['secret']
+
+ conf['_source_context'] = self.context
+ conf['_start_mark'] = self.start_mark
+
+ secret = configloader.SecretParser.fromYaml(layout, conf)
+ layout.addSecret(secret)
+
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
@@ -378,11 +420,11 @@
})
layout.addJob(in_repo_job_with_inherit_false)
- self.assertNotIn('auth', in_repo_job_without_inherit.auth)
- self.assertIn('secrets', in_repo_job_with_inherit.auth)
- self.assertEquals(in_repo_job_with_inherit.auth['secrets'],
- ['pypi-credentials'])
- self.assertNotIn('auth', in_repo_job_with_inherit_false.auth)
+ self.assertEqual(None, in_repo_job_without_inherit.auth)
+ self.assertEqual(1, len(in_repo_job_with_inherit.auth.secrets))
+ self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
+ 'pypi-credentials')
+ self.assertEqual(None, in_repo_job_with_inherit_false.auth)
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
@@ -537,6 +579,80 @@
"to shadow job base in base_project"):
layout.addJob(base2)
+ def test_job_allowed_projects(self):
+ job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
+ '_source_context': self.context,
+ '_start_mark': self.start_mark,
+ 'name': 'job',
+ 'allowed-projects': ['project'],
+ })
+ self.layout.addJob(job)
+
+ project2 = model.Project('project2', None)
+ context2 = model.SourceContext(project2, 'master',
+ 'test', True)
+
+ project2_config = configloader.ProjectParser.fromYaml(
+ self.tenant, self.layout, [{
+ '_source_context': context2,
+ '_start_mark': self.start_mark,
+ 'name': 'project2',
+ 'gate': {
+ 'jobs': [
+ 'job'
+ ]
+ }
+ }]
+ )
+ self.layout.addProjectConfig(project2_config)
+
+ change = model.Change(project2)
+ # Test master
+ change.branch = 'master'
+ item = self.queue.enqueueChange(change)
+ item.current_build_set.layout = self.layout
+ with testtools.ExpectedException(
+ Exception,
+ "Project project2 is not allowed to run job job"):
+ item.freezeJobGraph()
+
+ def test_job_pipeline_allow_secrets(self):
+ self.pipeline.allow_secrets = False
+ job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
+ '_source_context': self.context,
+ '_start_mark': self.start_mark,
+ 'name': 'job',
+ })
+ auth = model.AuthContext()
+ auth.secrets.append('foo')
+ job.auth = auth
+
+ self.layout.addJob(job)
+
+ project_config = configloader.ProjectParser.fromYaml(
+ self.tenant, self.layout, [{
+ '_source_context': self.context,
+ '_start_mark': self.start_mark,
+ 'name': 'project',
+ 'gate': {
+ 'jobs': [
+ 'job'
+ ]
+ }
+ }]
+ )
+ self.layout.addProjectConfig(project_config)
+
+ change = model.Change(self.project)
+ # Test master
+ change.branch = 'master'
+ item = self.queue.enqueueChange(change)
+ item.current_build_set.layout = self.layout
+ with testtools.ExpectedException(
+ Exception,
+ "Pipeline gate does not allow jobs with secrets"):
+ item.freezeJobGraph()
+
class TestJobTimeData(BaseTestCase):
def setUp(self):
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 8c5ef06..7de9be0 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -1950,28 +1950,25 @@
self.assertReportedStat('test-timing', '3|ms')
self.assertReportedStat('test-gauge', '12|g')
- @skip("Disabled for early v3 development")
def test_stuck_job_cleanup(self):
"Test that pending jobs are cleaned up if removed from layout"
- # This job won't be registered at startup because it is not in
- # the standard layout, but we need it to already be registerd
- # for when we reconfigure, as that is when Zuul will attempt
- # to run the new job.
- self.worker.registerFunction('build:gate-noop')
+
+ # We want to hold the project-merge job that the fake change enqueues
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
+ # The assertion is that we have one job in the queue, project-merge
self.assertEqual(len(self.gearman_server.getQueue()), 1)
- self.updateConfigLayout(
- 'tests/fixtures/layout-no-jobs.yaml')
+ self.commitLayoutUpdate('common-config', 'layout-no-jobs')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.gearman_server.release('gate-noop')
self.waitUntilSettled()
+ # asserting that project-merge is removed from queue
self.assertEqual(len(self.gearman_server.getQueue()), 0)
self.assertTrue(self.sched._areAllBuildsComplete())
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index fe8d560..678b957 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -20,7 +20,8 @@
import testtools
import zuul.configloader
-from tests.base import AnsibleZuulTestCase, ZuulTestCase
+from zuul.lib import encryption
+from tests.base import AnsibleZuulTestCase, ZuulTestCase, FIXTURE_DIR
class TestMultipleTenants(AnsibleZuulTestCase):
@@ -288,6 +289,11 @@
build.uuid + '.bare-role.flag')
self.assertTrue(os.path.exists(bare_role_flag_path))
+ secrets_path = os.path.join(self.test_root,
+ build.uuid + '.secrets')
+ with open(secrets_path) as f:
+ self.assertEqual(f.read(), "test-username test-password")
+
class TestBrokenConfig(ZuulTestCase):
# Test that we get an appropriate syntax error if we start with a
@@ -303,3 +309,33 @@
def test_broken_config_on_startup(self):
pass
+
+
+class TestProjectKeys(ZuulTestCase):
+ # Test that we can generate project keys
+
+ # Normally the test infrastructure copies a static key in place
+ # for each project before starting tests. This saves time because
+ # Zuul's automatic key-generation on startup can be slow. To make
+ # sure we exercise that code, in this test we allow Zuul to create
+ # keys for the project on startup.
+ create_project_keys = True
+ tenant_config_file = 'config/in-repo/main.yaml'
+
+ def test_key_generation(self):
+ key_root = os.path.join(self.state_root, 'keys')
+ private_key_file = os.path.join(key_root, 'gerrit/org/project.pem')
+ # Make sure that a proper key was created on startup
+ with open(private_key_file, "rb") as f:
+ private_key, public_key = \
+ encryption.deserialize_rsa_keypair(f.read())
+
+ with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
+ fixture_private_key = i.read()
+
+ # Make sure that we didn't just end up with the static fixture
+ # key
+ self.assertNotEqual(fixture_private_key, private_key)
+
+ # Make sure it's the right length
+ self.assertEqual(4096, private_key.key_size)
diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py
index acff09a..8791a25 100644
--- a/tests/unit/test_webapp.py
+++ b/tests/unit/test_webapp.py
@@ -15,11 +15,12 @@
# License for the specific language governing permissions and limitations
# under the License.
+import os
import json
from six.moves import urllib
-from tests.base import ZuulTestCase
+from tests.base import ZuulTestCase, FIXTURE_DIR
class TestWebapp(ZuulTestCase):
@@ -85,3 +86,13 @@
self.assertEqual(1, len(data), data)
self.assertEqual("org/project1", data[0]['project'], data)
+
+ def test_webapp_keys(self):
+ with open(os.path.join(FIXTURE_DIR, 'public.pem')) as f:
+ public_pem = f.read()
+
+ req = urllib.request.Request(
+ "http://localhost:%s/tenant-one/keys/gerrit/org/project.pub" %
+ self.port)
+ f = urllib.request.urlopen(req)
+ self.assertEqual(f.read(), public_pem)
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
new file mode 100644
index 0000000..4865edd
--- /dev/null
+++ b/tools/encrypt_secret.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+from six.moves import urllib
+
+DESCRIPTION = """Encrypt a secret for Zuul.
+
+This program fetches a project-specific public key from a Zuul server and
+uses that to encrypt a secret. The only pre-requisite is an installed
+OpenSSL binary.
+"""
+
+
+def main():
+ parser = argparse.ArgumentParser(description=DESCRIPTION)
+ parser.add_argument('url',
+ help="The base URL of the zuul server and tenant. "
+ "E.g., https://zuul.example.com/tenant-name")
+ # TODO(jeblair,mordred): When projects have canonical names, use that here.
+ # TODO(jeblair): Throw a fit if SSL is not used.
+ parser.add_argument('source',
+ help="The Zuul source of the project.")
+ parser.add_argument('project',
+ help="The name of the project.")
+ parser.add_argument('--infile',
+ default=None,
+ help="A filename whose contents will be encrypted. "
+ "If not supplied, the value will be read from "
+ "standard input.")
+ parser.add_argument('--outfile',
+ default=None,
+ help="A filename to which the encrypted value will be "
+ "written. If not supplied, the value will be written "
+ "to standard output.")
+ args = parser.parse_args()
+
+ req = urllib.request.Request("%s/keys/%s/%s.pub" % (
+ args.url, args.source, args.project))
+ pubkey = urllib.request.urlopen(req)
+
+ if args.infile:
+ with open(args.infile) as f:
+ plaintext = f.read()
+ else:
+ plaintext = sys.stdin.read()
+
+ pubkey_file = tempfile.NamedTemporaryFile(delete=False)
+ try:
+ pubkey_file.write(pubkey.read())
+ pubkey_file.close()
+
+ p = subprocess.Popen(['openssl', 'rsautl', '-encrypt',
+ '-oaep', '-pubin', '-inkey',
+ pubkey_file.name],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ (stdout, stderr) = p.communicate(plaintext)
+ if p.returncode != 0:
+ raise Exception("Return code %s from openssl" % p.returncode)
+ ciphertext = stdout.encode('base64')
+ finally:
+ os.unlink(pubkey_file.name)
+
+ if args.outfile:
+ with open(args.outfile, "w") as f:
+ f.write(ciphertext)
+ else:
+ print(ciphertext)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 8bae3c5..64c8db4 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -25,6 +25,7 @@
import zuul.manager.dependent
import zuul.manager.independent
from zuul import change_matcher
+from zuul.lib import encryption
# Several forms accept either a single item or a list, this makes
@@ -84,7 +85,7 @@
class ZuulSafeLoader(yaml.SafeLoader):
- zuul_node_types = frozenset(('job', 'nodeset', 'pipeline',
+ zuul_node_types = frozenset(('job', 'nodeset', 'secret', 'pipeline',
'project', 'project-template'))
def __init__(self, stream, context):
@@ -122,6 +123,29 @@
loader.dispose()
+class EncryptedPKCS1_OAEP(yaml.YAMLObject):
+ yaml_tag = u'!encrypted/pkcs1-oaep'
+ yaml_loader = yaml.SafeLoader
+
+ def __init__(self, ciphertext):
+ self.ciphertext = ciphertext.decode('base64')
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, EncryptedPKCS1_OAEP):
+ return False
+ return (self.ciphertext == other.ciphertext)
+
+ @classmethod
+ def from_yaml(cls, loader, node):
+ return cls(node.value)
+
+ def decrypt(self, private_key):
+ return encryption.decrypt_pkcs1_oaep(self.ciphertext, private_key)
+
+
class NodeSetParser(object):
@staticmethod
def getSchema():
@@ -148,6 +172,28 @@
return ns
+class SecretParser(object):
+ @staticmethod
+ def getSchema():
+ data = {str: vs.Any(str, EncryptedPKCS1_OAEP)}
+
+ secret = {vs.Required('name'): str,
+ vs.Required('data'): data,
+ '_source_context': model.SourceContext,
+ '_start_mark': yaml.Mark,
+ }
+
+ return vs.Schema(secret)
+
+ @staticmethod
+ def fromYaml(layout, conf):
+ with configuration_exceptions('secret', conf):
+ SecretParser.getSchema()(conf)
+ s = model.Secret(conf['name'], conf['_source_context'])
+ s.secret_data = conf['data']
+ return s
+
+
class JobParser(object):
@staticmethod
def getSchema():
@@ -194,6 +240,7 @@
'repos': to_list(str),
'vars': dict,
'dependencies': to_list(str),
+ 'allowed-projects': to_list(str),
}
return vs.Schema(job)
@@ -224,7 +271,19 @@
job = model.Job(conf['name'])
job.source_context = conf.get('_source_context')
if 'auth' in conf:
- job.auth = conf.get('auth')
+ job.auth = model.AuthContext()
+ if 'inherit' in conf['auth']:
+ job.auth.inherit = conf['auth']['inherit']
+
+ for secret_name in conf['auth'].get('secrets', []):
+ secret = layout.secrets[secret_name]
+ if secret.source_context != job.source_context:
+ raise Exception(
+ "Unable to use secret %s. Secrets must be "
+ "defined in the same project in which they "
+ "are used" % secret_name)
+ job.auth.secrets.append(secret.decrypt(
+ job.source_context.project.private_key))
if 'parent' in conf:
parent = layout.getJob(conf['parent'])
@@ -291,6 +350,19 @@
if variables:
job.updateVariables(variables)
+ allowed_projects = conf.get('allowed-projects', None)
+ if allowed_projects:
+ allowed = []
+ for p in as_list(allowed_projects):
+ # TODOv3(jeblair): this limits allowed_projects to the same
+ # source; we should remove that limitation.
+ source = job.source_context.project.connection_name
+ (trusted, project) = tenant.getRepo(source, p)
+ if project is None:
+ raise Exception("Unknown project %s" % (p,))
+ allowed.append(project.name)
+ job.allowed_projects = frozenset(allowed)
+
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
if (not job.source_context.trusted):
@@ -465,6 +537,7 @@
project_pipeline.queue_name = queue_name
if pipeline_defined:
project.pipelines[pipeline.name] = project_pipeline
+
return project
@@ -535,6 +608,7 @@
'footer-message': str,
'dequeue-on-new-patchset': bool,
'ignore-dependencies': bool,
+ 'allow-secrets': bool,
'disable-after-consecutive-failures':
vs.All(int, vs.Range(min=1)),
'window': window,
@@ -582,6 +656,7 @@
'dequeue-on-new-patchset', True)
pipeline.ignore_dependencies = conf.get(
'ignore-dependencies', False)
+ pipeline.allow_secrets = conf.get('allow-secrets', False)
for conf_key, action in PipelineParser.reporter_actions.items():
reporter_set = []
@@ -673,13 +748,15 @@
return vs.Schema(tenant)
@staticmethod
- def fromYaml(base, connections, scheduler, merger, conf, cached):
+ def fromYaml(base, project_key_dir, connections, scheduler, merger, conf,
+ cached):
TenantParser.getSchema(connections)(conf)
tenant = model.Tenant(conf['name'])
tenant.unparsed_config = conf
unparsed_config = model.UnparsedTenantConfig()
tenant.config_repos, tenant.project_repos = \
- TenantParser._loadTenantConfigRepos(connections, conf)
+ TenantParser._loadTenantConfigRepos(
+ project_key_dir, connections, conf)
for source, repo in tenant.config_repos:
tenant.addConfigRepo(source, repo)
for source, repo in tenant.project_repos:
@@ -699,7 +776,53 @@
return tenant
@staticmethod
- def _loadTenantConfigRepos(connections, conf_tenant):
+ def _loadProjectKeys(project_key_dir, connection_name, project):
+ project.private_key_file = (
+ os.path.join(project_key_dir, connection_name,
+ project.name + '.pem'))
+
+ TenantParser._generateKeys(project)
+ TenantParser._loadKeys(project)
+
+ @staticmethod
+ def _generateKeys(project):
+ if os.path.isfile(project.private_key_file):
+ return
+
+ key_dir = os.path.dirname(project.private_key_file)
+ if not os.path.isdir(key_dir):
+ os.makedirs(key_dir)
+
+ TenantParser.log.info(
+ "Generating RSA keypair for project %s" % (project.name,)
+ )
+ private_key, public_key = encryption.generate_rsa_keypair()
+ pem_private_key = encryption.serialize_rsa_private_key(private_key)
+
+ # Dump keys to filesystem. We only save the private key
+ # because the public key can be constructed from it.
+ TenantParser.log.info(
+ "Saving RSA keypair for project %s to %s" % (
+ project.name, project.private_key_file)
+ )
+ with open(project.private_key_file, 'wb') as f:
+ f.write(pem_private_key)
+
+ @staticmethod
+ def _loadKeys(project):
+ # Check the key files specified are there
+ if not os.path.isfile(project.private_key_file):
+ raise Exception(
+ 'Private key file {0} not found'.format(
+ project.private_key_file))
+
+ # Load keypair
+ with open(project.private_key_file, "rb") as f:
+ (project.private_key, project.public_key) = \
+ encryption.deserialize_rsa_keypair(f.read())
+
+ @staticmethod
+ def _loadTenantConfigRepos(project_key_dir, connections, conf_tenant):
config_repos = []
project_repos = []
@@ -708,10 +831,14 @@
for conf_repo in conf_source.get('config-repos', []):
project = source.getProject(conf_repo)
+ TenantParser._loadProjectKeys(
+ project_key_dir, source_name, project)
config_repos.append((source, project))
for conf_repo in conf_source.get('project-repos', []):
project = source.getProject(conf_repo)
+ TenantParser._loadProjectKeys(
+ project_key_dir, source_name, project)
project_repos.append((source, project))
return config_repos, project_repos
@@ -833,6 +960,9 @@
for config_nodeset in data.nodesets:
layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
+ for config_secret in data.secrets:
+ layout.addSecret(SecretParser.fromYaml(layout, config_secret))
+
for config_job in data.jobs:
layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
@@ -861,7 +991,8 @@
config_path)
return config_path
- def loadConfig(self, config_path, scheduler, merger, connections):
+ def loadConfig(self, config_path, project_key_dir, scheduler, merger,
+ connections):
abide = model.Abide()
config_path = self.expandConfigPath(config_path)
@@ -874,13 +1005,14 @@
for conf_tenant in config.tenants:
# When performing a full reload, do not use cached data.
- tenant = TenantParser.fromYaml(base, connections, scheduler,
- merger, conf_tenant, cached=False)
+ tenant = TenantParser.fromYaml(
+ base, project_key_dir, connections, scheduler, merger,
+ conf_tenant, cached=False)
abide.tenants[tenant.name] = tenant
return abide
- def reloadTenant(self, config_path, scheduler, merger, connections,
- abide, tenant):
+ def reloadTenant(self, config_path, project_key_dir, scheduler,
+ merger, connections, abide, tenant):
new_abide = model.Abide()
new_abide.tenants = abide.tenants.copy()
@@ -888,9 +1020,9 @@
base = os.path.dirname(os.path.realpath(config_path))
# When reloading a tenant only, use cached data if available.
- new_tenant = TenantParser.fromYaml(base, connections, scheduler,
- merger, tenant.unparsed_config,
- cached=True)
+ new_tenant = TenantParser.fromYaml(
+ base, project_key_dir, connections, scheduler, merger,
+ tenant.unparsed_config, cached=True)
new_abide.tenants[tenant.name] = new_tenant
return new_abide
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 514aa1f..e3c726f 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -93,6 +93,9 @@
event.ref = refupdate.get('refName')
event.oldrev = refupdate.get('oldRev')
event.newrev = refupdate.get('newRev')
+ if event.project_name is None:
+ # ref-replica* events
+ event.project_name = data.get('project')
# Map the event types to a field name holding a Gerrit
# account attribute. See Gerrit stream-event documentation
# in cmd-stream-events.html
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 220f82a..90cfa9b 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -311,6 +311,9 @@
public_ipv4=node.public_ipv4))
params['nodes'] = nodes
params['vars'] = copy.deepcopy(job.variables)
+ if job.auth:
+ for secret in job.auth.secrets:
+ params['vars'][secret.name] = copy.deepcopy(secret.secret_data)
params['vars']['zuul'] = zuul_params
projects = set()
if job.repos:
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 60b30c7..67fc5e6 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -646,10 +646,15 @@
nodepool_az=node.get('az'),
nodepool_provider=node.get('provider'),
nodepool_region=node.get('region'))
+
+ host_keys = []
+ for key in node.get('host_keys'):
+ host_keys.append("%s %s" % (ip, key))
+
hosts.append(dict(
name=node['name'],
host_vars=host_vars,
- host_keys=node.get('host_keys')))
+ host_keys=host_keys))
return hosts
def _blockPluginDirs(self, path):
diff --git a/zuul/lib/encryption.py b/zuul/lib/encryption.py
new file mode 100644
index 0000000..24224d8
--- /dev/null
+++ b/zuul/lib/encryption.py
@@ -0,0 +1,138 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import padding
+from cryptography.hazmat.primitives import hashes
+
+
+# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#generation
+def generate_rsa_keypair():
+ """Generate an RSA keypair.
+
+ :returns: A tuple (private_key, public_key)
+
+ """
+ private_key = rsa.generate_private_key(
+ public_exponent=65537,
+ key_size=4096,
+ backend=default_backend()
+ )
+ public_key = private_key.public_key()
+ return (private_key, public_key)
+
+
+# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#key-serialization
+def serialize_rsa_private_key(private_key):
+ """Serialize an RSA private key
+
+ This returns a PEM-encoded serialized form of an RSA private key
+ suitable for storing on disk. It is not password-protected.
+
+ :arg private_key: A private key object as returned by
+ :func:generate_rsa_keypair()
+
+ :returns: A PEM-encoded string representation of the private key.
+
+ """
+ return private_key.private_bytes(
+ encoding=serialization.Encoding.PEM,
+ format=serialization.PrivateFormat.TraditionalOpenSSL,
+ encryption_algorithm=serialization.NoEncryption()
+ )
+
+
+def serialize_rsa_public_key(public_key):
+ """Serialize an RSA public key
+
+ This returns a PEM-encoded serialized form of an RSA public key
+ suitable for distribution.
+
+ :arg public_key: A pubilc key object as returned by
+ :func:generate_rsa_keypair()
+
+ :returns: A PEM-encoded string representation of the public key.
+
+ """
+ return public_key.public_bytes(
+ encoding=serialization.Encoding.PEM,
+ format=serialization.PublicFormat.SubjectPublicKeyInfo
+ )
+
+
+# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#key-loading
+def deserialize_rsa_keypair(data):
+ """Deserialize an RSA private key
+
+ This deserializes an RSA private key and returns the keypair
+ (private and public) for use in decryption.
+
+ :arg data: A PEM-encoded serialized private key
+
+ :returns: A tuple (private_key, public_key)
+
+ """
+ private_key = serialization.load_pem_private_key(
+ data,
+ password=None,
+ backend=default_backend()
+ )
+ public_key = private_key.public_key()
+ return (private_key, public_key)
+
+
+# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#decryption
+def decrypt_pkcs1_oaep(ciphertext, private_key):
+ """Decrypt PKCS#1 (RSAES-OAEP) encoded ciphertext
+
+ :arg ciphertext: A string previously encrypted with PKCS#1
+ (RSAES-OAEP).
+ :arg private_key: A private key object as returned by
+ :func:generate_rsa_keypair()
+
+ :returns: The decrypted form of the ciphertext as a string.
+
+ """
+ return private_key.decrypt(
+ ciphertext,
+ padding.OAEP(
+ mgf=padding.MGF1(algorithm=hashes.SHA1()),
+ algorithm=hashes.SHA1(),
+ label=None
+ )
+ )
+
+
+# https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#encryption
+def encrypt_pkcs1_oaep(plaintext, public_key):
+ """Encrypt data with PKCS#1 (RSAES-OAEP)
+
+ :arg plaintext: A string to encrypt with PKCS#1 (RSAES-OAEP).
+
+ :arg public_key: A public key object as returned by
+ :func:generate_rsa_keypair()
+
+ :returns: The encrypted form of the plaintext.
+
+ """
+ return public_key.encrypt(
+ plaintext,
+ padding.OAEP(
+ mgf=padding.MGF1(algorithm=hashes.SHA1()),
+ algorithm=hashes.SHA1(),
+ label=None
+ )
+ )
diff --git a/zuul/model.py b/zuul/model.py
index afa7522..a4ac278 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -128,6 +128,7 @@
self.success_message = None
self.footer_message = None
self.start_message = None
+ self.allow_secrets = False
self.dequeue_on_new_patchset = True
self.ignore_dependencies = False
self.manager = None
@@ -522,6 +523,51 @@
self.state_time = data['state_time']
+class Secret(object):
+ """A collection of private data.
+
+ In configuration, Secrets are collections of private data in
+ key-value pair format. They are defined as top-level
+ configuration objects and then referenced by Jobs.
+
+ """
+
+ def __init__(self, name, source_context):
+ self.name = name
+ self.source_context = source_context
+ # The secret data may or may not be encrypted. This attribute
+ # is named 'secret_data' to make it easy to search for and
+ # spot where it is directly used.
+ self.secret_data = {}
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, Secret):
+ return False
+ return (self.name == other.name and
+ self.source_context == other.source_context and
+ self.secret_data == other.secret_data)
+
+ def __repr__(self):
+ return '<Secret %s>' % (self.name,)
+
+ def decrypt(self, private_key):
+ """Return a copy of this secret with any encrypted data decrypted.
+ Note that the original remains encrypted."""
+
+ r = copy.deepcopy(self)
+ decrypted_secret_data = {}
+ for k, v in r.secret_data.items():
+ if hasattr(v, 'decrypt'):
+ decrypted_secret_data[k] = v.decrypt(private_key)
+ else:
+ decrypted_secret_data[k] = v
+ r.secret_data = decrypted_secret_data
+ return r
+
+
class SourceContext(object):
"""A reference to the branch of a project in configuration.
@@ -651,6 +697,28 @@
return d
+class AuthContext(object):
+ """The authentication information for a job.
+
+ Authentication information (both the actual data and metadata such
+ as whether it should be inherited) for a job is grouped together
+ in this object.
+ """
+
+ def __init__(self, inherit=False):
+ self.inherit = inherit
+ self.secrets = []
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, AuthContext):
+ return False
+ return (self.inherit == other.inherit and
+ self.secrets == other.secrets)
+
+
class Job(object):
"""A Job represents the defintion of actions to perform.
@@ -693,7 +761,7 @@
timeout=None,
variables={},
nodeset=NodeSet(),
- auth={},
+ auth=None,
workspace=None,
pre_run=(),
post_run=(),
@@ -704,6 +772,7 @@
final=False,
roles=frozenset(),
repos=frozenset(),
+ allowed_projects=None,
)
# These are generally internal attributes which are not
@@ -783,7 +852,7 @@
raise Exception("Job unable to inherit from %s" % (other,))
do_not_inherit = set()
- if other.auth and not other.auth.get('inherit'):
+ if other.auth and not other.auth.inherit:
do_not_inherit.add('auth')
# copy all attributes
@@ -2082,6 +2151,7 @@
self.name = name
self.merge_mode = None
self.pipelines = {}
+ self.private_key_file = None
class UnparsedAbideConfig(object):
@@ -2129,6 +2199,7 @@
self.project_templates = []
self.projects = {}
self.nodesets = []
+ self.secrets = []
def copy(self):
r = UnparsedTenantConfig()
@@ -2137,6 +2208,7 @@
r.project_templates = copy.deepcopy(self.project_templates)
r.projects = copy.deepcopy(self.projects)
r.nodesets = copy.deepcopy(self.nodesets)
+ r.secrets = copy.deepcopy(self.secrets)
return r
def extend(self, conf):
@@ -2147,6 +2219,7 @@
for k, v in conf.projects.items():
self.projects.setdefault(k, []).extend(v)
self.nodesets.extend(conf.nodesets)
+ self.secrets.extend(conf.secrets)
return
if not isinstance(conf, list):
@@ -2175,6 +2248,8 @@
self.pipelines.append(value)
elif key == 'nodeset':
self.nodesets.append(value)
+ elif key == 'secret':
+ self.secrets.append(value)
else:
raise Exception("Configuration item `%s` not recognized "
"(when parsing %s)" %
@@ -2197,6 +2272,7 @@
# inherit from the reference definition.
self.jobs = {'noop': [Job('noop')]}
self.nodesets = {}
+ self.secrets = {}
def getJob(self, name):
if name in self.jobs:
@@ -2230,6 +2306,11 @@
raise Exception("NodeSet %s already defined" % (nodeset.name,))
self.nodesets[nodeset.name] = nodeset
+ def addSecret(self, secret):
+ if secret.name in self.secrets:
+ raise Exception("Secret %s already defined" % (secret.name,))
+ self.secrets[secret.name] = secret
+
def addPipeline(self, pipeline):
self.pipelines[pipeline.name] = pipeline
@@ -2239,7 +2320,9 @@
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
- def _createJobGraph(self, change, job_list, job_graph):
+ def _createJobGraph(self, item, job_list, job_graph):
+ change = item.change
+ pipeline = item.pipeline
for jobname in job_list.jobs:
# This is the final job we are constructing
frozen_job = None
@@ -2261,7 +2344,7 @@
# If the job does not allow auth inheritance, do not allow
# the project-pipeline variants to update its execution
# attributes.
- if frozen_job.auth and not frozen_job.auth.get('inherit'):
+ if frozen_job.auth and not frozen_job.auth.inherit:
frozen_job.final = True
# Whether the change matches any of the project pipeline
# variants
@@ -2274,6 +2357,15 @@
# A change must match at least one project pipeline
# job variant.
continue
+ if (frozen_job.allowed_projects and
+ change.project.name not in frozen_job.allowed_projects):
+ raise Exception("Project %s is not allowed to run job %s" %
+ (change.project.name, frozen_job.name))
+ if ((not pipeline.allow_secrets) and frozen_job.auth and
+ frozen_job.auth.secrets):
+ raise Exception("Pipeline %s does not allow jobs with "
+ "secrets (job %s)" % (
+ pipeline.name, frozen_job.name))
job_graph.addJob(frozen_job)
def createJobGraph(self, item):
@@ -2285,7 +2377,7 @@
if project_config and item.pipeline.name in project_config.pipelines:
project_job_list = \
project_config.pipelines[item.pipeline.name].job_list
- self._createJobGraph(item.change, project_job_list, ret)
+ self._createJobGraph(item, project_job_list, ret)
return ret
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index dae9fe7..952ad9a 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -452,6 +452,22 @@
os.mkdir(d)
return d
+ def _get_project_key_dir(self):
+ if self.config.has_option('zuul', 'state_dir'):
+ state_dir = os.path.expanduser(self.config.get('zuul',
+ 'state_dir'))
+ else:
+ state_dir = '/var/lib/zuul'
+ key_dir = os.path.join(state_dir, 'keys')
+ if not os.path.exists(key_dir):
+ os.mkdir(key_dir, 0o700)
+ st = os.stat(key_dir)
+ mode = st.st_mode & 0o777
+ if mode != 0o700:
+ raise Exception("Project key directory %s must be mode 0700; "
+ "current mode is %o" % (key_dir, mode))
+ return key_dir
+
def _save_queue(self):
pickle_file = self._get_queue_pickle_file()
events = []
@@ -507,6 +523,7 @@
loader = configloader.ConfigLoader()
abide = loader.loadConfig(
self.config.get('zuul', 'tenant_config'),
+ self._get_project_key_dir(),
self, self.merger, self.connections)
for tenant in abide.tenants.values():
self._reconfigureTenant(tenant)
@@ -523,6 +540,7 @@
loader = configloader.ConfigLoader()
abide = loader.reloadTenant(
self.config.get('zuul', 'tenant_config'),
+ self._get_project_key_dir(),
self, self.merger, self.connections,
self.abide, event.tenant)
tenant = abide.tenants[event.tenant.name]
diff --git a/zuul/webapp.py b/zuul/webapp.py
index e16f0b4..4f040fa 100644
--- a/zuul/webapp.py
+++ b/zuul/webapp.py
@@ -23,6 +23,8 @@
import webob
from webob import dec
+from zuul.lib import encryption
+
"""Zuul main web app.
Zuul supports HTTP requests directly against it for determining the
@@ -34,6 +36,7 @@
queue / pipeline structure of the system
- /status.json (backwards compatibility): same as /status
- /status/change/X,Y: return status just for gerrit change X,Y
+ - /keys/SOURCE/PROJECT.pub: return the public key for PROJECT
When returning status for a single gerrit change you will get an
array of changes, they will not include the queue structure.
@@ -96,9 +99,31 @@
return m.group(1)
return None
+ def _handle_keys(self, request, path):
+ m = re.match('/keys/(.*?)/(.*?).pub', path)
+ if not m:
+ raise webob.exc.HTTPNotFound()
+ source_name = m.group(1)
+ project_name = m.group(2)
+ source = self.scheduler.connections.getSource(source_name)
+ if not source:
+ raise webob.exc.HTTPNotFound()
+ project = source.getProject(project_name)
+ if not project:
+ raise webob.exc.HTTPNotFound()
+
+ pem_public_key = encryption.serialize_rsa_public_key(
+ project.public_key)
+
+ response = webob.Response(body=pem_public_key,
+ content_type='text/plain')
+ return response.conditional_response_app
+
def app(self, request):
tenant_name = request.path.split('/')[1]
path = request.path.replace('/' + tenant_name, '')
+ if path.startswith('/keys'):
+ return self._handle_keys(request, path)
path = self._normalize_path(path)
if path is None:
raise webob.exc.HTTPNotFound()