Add secret top-level config object
This adds secrets as a top-level config object, including a new
custom YAML tag to indicate encrypted data.
It also adds a script which encrypts data for use in tests.
Change-Id: I92a6bc048874f8aa4ebe0dd27180b253bede7370
diff --git a/tests/encrypt_secret.py b/tests/encrypt_secret.py
new file mode 100644
index 0000000..ab45018
--- /dev/null
+++ b/tests/encrypt_secret.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+import os
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import padding
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives import hashes
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+ 'fixtures')
+
+
+def main():
+ private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
+ with open(private_key_file, "rb") as f:
+ private_key = serialization.load_pem_private_key(
+ f.read(),
+ password=None,
+ backend=default_backend()
+ )
+
+ # Extract public key from private
+ public_key = private_key.public_key()
+
+ # https://cryptography.io/en/stable/hazmat/primitives/asymmetric/rsa/#encryption
+ ciphertext = public_key.encrypt(
+ sys.argv[1],
+ padding.OAEP(
+ mgf=padding.MGF1(algorithm=hashes.SHA1()),
+ algorithm=hashes.SHA1(),
+ label=None
+ )
+ )
+ print(ciphertext.encode('base64'))
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index 50f353d..eb3dbd8 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -34,6 +34,21 @@
verified: 0
precedence: high
+- secret:
+ name: test_secret
+ data:
+ username: test-username
+ password: !encrypted/pkcs1 |
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+ ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+ 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+ Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+ xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+ aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+ +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
- job:
name: python27
pre-run: pre
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index ee7c6ab..335d7c3 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -302,6 +302,29 @@
tenant = model.Tenant('tenant')
layout = model.Layout()
+ conf = yaml.safe_load('''
+- secret:
+ name: pypi-credentials
+ data:
+ username: test-username
+ password: !encrypted/pkcs1 |
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+ ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+ 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+ Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+ xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+ aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+ +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+''')[0]['secret']
+
+ conf['_source_context'] = self.context
+ conf['_start_mark'] = self.start_mark
+
+ secret = configloader.SecretParser.fromYaml(layout, conf)
+ layout.addSecret(secret)
+
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 8439fc3..ae980ac 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -87,7 +87,7 @@
class ZuulSafeLoader(yaml.SafeLoader):
- zuul_node_types = frozenset(('job', 'nodeset', 'pipeline',
+ zuul_node_types = frozenset(('job', 'nodeset', 'secret', 'pipeline',
'project', 'project-template'))
def __init__(self, stream, context):
@@ -125,6 +125,18 @@
loader.dispose()
+class EncryptedPKCS1(yaml.YAMLObject):
+ yaml_tag = u'!encrypted/pkcs1'
+ yaml_loader = yaml.SafeLoader
+
+ def __init__(self, ciphertext):
+ self.ciphertext = ciphertext
+
+ @classmethod
+ def from_yaml(cls, loader, node):
+ return cls(node.value)
+
+
class NodeSetParser(object):
@staticmethod
def getSchema():
@@ -151,6 +163,28 @@
return ns
+class SecretParser(object):
+ @staticmethod
+ def getSchema():
+ data = {str: vs.Any(str, EncryptedPKCS1)}
+
+ secret = {vs.Required('name'): str,
+ vs.Required('data'): data,
+ '_source_context': model.SourceContext,
+ '_start_mark': yaml.Mark,
+ }
+
+ return vs.Schema(secret)
+
+ @staticmethod
+ def fromYaml(layout, conf):
+ with configuration_exceptions('secret', conf):
+ SecretParser.getSchema()(conf)
+ s = model.Secret(conf['name'])
+ s.secret_data = conf['data']
+ return s
+
+
class JobParser(object):
@staticmethod
def getSchema():
@@ -906,6 +940,9 @@
for config_nodeset in data.nodesets:
layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
+ for config_secret in data.secrets:
+ layout.addSecret(SecretParser.fromYaml(layout, config_secret))
+
for config_job in data.jobs:
layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
diff --git a/zuul/model.py b/zuul/model.py
index 9c395c4..e929f8b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -515,6 +515,35 @@
self.state_time = data['state_time']
+class Secret(object):
+ """A collection of private data.
+
+ In configuration, Secrets are collections of private data in
+ key-value pair format. They are defined as top-level
+ configuration objects and then referenced by Jobs.
+
+ """
+
+ def __init__(self, name):
+ self.name = name
+ # The secret data may or may not be encrypted. This attribute
+ # is named 'secret_data' to make it easy to search for and
+ # spot where it is directly used.
+ self.secret_data = {}
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ if not isinstance(other, Secret):
+ return False
+ return (self.name == other.name and
+ self.secret_data == other.secret_data)
+
+ def __repr__(self):
+ return '<Secret %s>' % (self.name,)
+
+
class SourceContext(object):
"""A reference to the branch of a project in configuration.
@@ -2132,6 +2161,7 @@
self.project_templates = []
self.projects = {}
self.nodesets = []
+ self.secrets = []
def copy(self):
r = UnparsedTenantConfig()
@@ -2140,6 +2170,7 @@
r.project_templates = copy.deepcopy(self.project_templates)
r.projects = copy.deepcopy(self.projects)
r.nodesets = copy.deepcopy(self.nodesets)
+ r.secrets = copy.deepcopy(self.secrets)
return r
def extend(self, conf):
@@ -2150,6 +2181,7 @@
for k, v in conf.projects.items():
self.projects.setdefault(k, []).extend(v)
self.nodesets.extend(conf.nodesets)
+ self.secrets.extend(conf.secrets)
return
if not isinstance(conf, list):
@@ -2178,6 +2210,8 @@
self.pipelines.append(value)
elif key == 'nodeset':
self.nodesets.append(value)
+ elif key == 'secret':
+ self.secrets.append(value)
else:
raise Exception("Configuration item `%s` not recognized "
"(when parsing %s)" %
@@ -2200,6 +2234,7 @@
# inherit from the reference definition.
self.jobs = {'noop': [Job('noop')]}
self.nodesets = {}
+ self.secrets = {}
def getJob(self, name):
if name in self.jobs:
@@ -2233,6 +2268,11 @@
raise Exception("NodeSet %s already defined" % (nodeset.name,))
self.nodesets[nodeset.name] = nodeset
+ def addSecret(self, secret):
+ if secret.name in self.secrets:
+ raise Exception("Secret %s already defined" % (secret.name,))
+ self.secrets[secret.name] = secret
+
def addPipeline(self, pipeline):
self.pipelines[pipeline.name] = pipeline