From 01f83b73f27997d534a0a13f29d051143a32a598 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 15 Mar 2017 13:03:40 -0700 Subject: [PATCH] Add secret top-level config object This adds secrets as a top-level config object, including a new custom YAML tag to indicate encrypted data. It also adds a script which encrypts data for use in tests. Change-Id: I92a6bc048874f8aa4ebe0dd27180b253bede7370 --- tests/encrypt_secret.py | 51 +++++++++++++++++++ .../ansible/git/common-config/zuul.yaml | 15 ++++++ tests/unit/test_model.py | 23 +++++++++ zuul/configloader.py | 39 +++++++++++++- zuul/model.py | 40 +++++++++++++++ 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 tests/encrypt_secret.py diff --git a/tests/encrypt_secret.py b/tests/encrypt_secret.py new file mode 100644 index 0000000000..ab45018d51 --- /dev/null +++ b/tests/encrypt_secret.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import os + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'fixtures') + + +def main(): + private_key_file = os.path.join(FIXTURE_DIR, 'private.pem') + with open(private_key_file, "rb") as f: + private_key = serialization.load_pem_private_key( + f.read(), + password=None, + backend=default_backend() + ) + + # Extract public key from private + public_key = private_key.public_key() + + # https://cryptography.io/en/stable/hazmat/primitives/asymmetric/rsa/#encryption + ciphertext = public_key.encrypt( + sys.argv[1], + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA1()), + algorithm=hashes.SHA1(), + label=None + ) + ) + print(ciphertext.encode('base64')) + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml index 50f353d5e5..eb3dbd80ca 100644 --- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml @@ -34,6 +34,21 @@ verified: 0 precedence: high +- secret: + name: test_secret + data: + username: test-username + password: !encrypted/pkcs1 | + BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ + L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o + ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+ + 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v + Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt + xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr + aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW + Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd + +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs= + - job: name: python27 pre-run: pre diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index ee7c6abe7e..335d7c3035 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -302,6 +302,29 @@ class TestJob(BaseTestCase): tenant = model.Tenant('tenant') layout = model.Layout() + conf = yaml.safe_load(''' +- secret: + name: pypi-credentials + data: + username: test-username + password: !encrypted/pkcs1 | + BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ + L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o + ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+ + 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v + Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt + xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr + aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW + Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd + +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs= +''')[0]['secret'] + + conf['_source_context'] = self.context + conf['_start_mark'] = self.start_mark + + secret = configloader.SecretParser.fromYaml(layout, conf) + layout.addSecret(secret) + base = configloader.JobParser.fromYaml(tenant, layout, { '_source_context': self.context, '_start_mark': self.start_mark, diff --git a/zuul/configloader.py b/zuul/configloader.py index 8439fc36eb..ae980ac7d2 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -87,7 +87,7 @@ def configuration_exceptions(stanza, conf): class ZuulSafeLoader(yaml.SafeLoader): - zuul_node_types = frozenset(('job', 'nodeset', 'pipeline', + zuul_node_types = frozenset(('job', 'nodeset', 'secret', 'pipeline', 'project', 'project-template')) def __init__(self, stream, context): @@ -125,6 +125,18 @@ repo {repo} on branch {branch}. The error was: loader.dispose() +class EncryptedPKCS1(yaml.YAMLObject): + yaml_tag = u'!encrypted/pkcs1' + yaml_loader = yaml.SafeLoader + + def __init__(self, ciphertext): + self.ciphertext = ciphertext + + @classmethod + def from_yaml(cls, loader, node): + return cls(node.value) + + class NodeSetParser(object): @staticmethod def getSchema(): @@ -151,6 +163,28 @@ class NodeSetParser(object): return ns +class SecretParser(object): + @staticmethod + def getSchema(): + data = {str: vs.Any(str, EncryptedPKCS1)} + + secret = {vs.Required('name'): str, + vs.Required('data'): data, + '_source_context': model.SourceContext, + '_start_mark': yaml.Mark, + } + + return vs.Schema(secret) + + @staticmethod + def fromYaml(layout, conf): + with configuration_exceptions('secret', conf): + SecretParser.getSchema()(conf) + s = model.Secret(conf['name']) + s.secret_data = conf['data'] + return s + + class JobParser(object): @staticmethod def getSchema(): @@ -906,6 +940,9 @@ class TenantParser(object): for config_nodeset in data.nodesets: layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset)) + for config_secret in data.secrets: + layout.addSecret(SecretParser.fromYaml(layout, config_secret)) + for config_job in data.jobs: layout.addJob(JobParser.fromYaml(tenant, layout, config_job)) diff --git a/zuul/model.py b/zuul/model.py index 9c395c4654..e929f8b92a 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -515,6 +515,35 @@ class NodeRequest(object): self.state_time = data['state_time'] +class Secret(object): + """A collection of private data. + + In configuration, Secrets are collections of private data in + key-value pair format. They are defined as top-level + configuration objects and then referenced by Jobs. + + """ + + def __init__(self, name): + self.name = name + # The secret data may or may not be encrypted. This attribute + # is named 'secret_data' to make it easy to search for and + # spot where it is directly used. + self.secret_data = {} + + def __ne__(self, other): + return not self.__eq__(other) + + def __eq__(self, other): + if not isinstance(other, Secret): + return False + return (self.name == other.name and + self.secret_data == other.secret_data) + + def __repr__(self): + return '' % (self.name,) + + class SourceContext(object): """A reference to the branch of a project in configuration. @@ -2132,6 +2161,7 @@ class UnparsedTenantConfig(object): self.project_templates = [] self.projects = {} self.nodesets = [] + self.secrets = [] def copy(self): r = UnparsedTenantConfig() @@ -2140,6 +2170,7 @@ class UnparsedTenantConfig(object): r.project_templates = copy.deepcopy(self.project_templates) r.projects = copy.deepcopy(self.projects) r.nodesets = copy.deepcopy(self.nodesets) + r.secrets = copy.deepcopy(self.secrets) return r def extend(self, conf): @@ -2150,6 +2181,7 @@ class UnparsedTenantConfig(object): for k, v in conf.projects.items(): self.projects.setdefault(k, []).extend(v) self.nodesets.extend(conf.nodesets) + self.secrets.extend(conf.secrets) return if not isinstance(conf, list): @@ -2178,6 +2210,8 @@ class UnparsedTenantConfig(object): self.pipelines.append(value) elif key == 'nodeset': self.nodesets.append(value) + elif key == 'secret': + self.secrets.append(value) else: raise Exception("Configuration item `%s` not recognized " "(when parsing %s)" % @@ -2200,6 +2234,7 @@ class Layout(object): # inherit from the reference definition. self.jobs = {'noop': [Job('noop')]} self.nodesets = {} + self.secrets = {} def getJob(self, name): if name in self.jobs: @@ -2233,6 +2268,11 @@ class Layout(object): raise Exception("NodeSet %s already defined" % (nodeset.name,)) self.nodesets[nodeset.name] = nodeset + def addSecret(self, secret): + if secret.name in self.secrets: + raise Exception("Secret %s already defined" % (secret.name,)) + self.secrets[secret.name] = secret + def addPipeline(self, pipeline): self.pipelines[pipeline.name] = pipeline