Add secret top-level config object

This adds secrets as a top-level config object, including a new
custom YAML tag to indicate encrypted data.

It also adds a script which encrypts data for use in tests.

Change-Id: I92a6bc048874f8aa4ebe0dd27180b253bede7370

tests/encrypt_secret.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+import os
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import padding
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives import hashes
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+                           'fixtures')
+
+
+def main():
+    private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
+    with open(private_key_file, "rb") as f:
+        private_key = serialization.load_pem_private_key(
+            f.read(),
+            password=None,
+            backend=default_backend()
+        )
+
+    # Extract public key from private
+    public_key = private_key.public_key()
+
+    # https://cryptography.io/en/stable/hazmat/primitives/asymmetric/rsa/#encryption
+    ciphertext = public_key.encrypt(
+        sys.argv[1],
+        padding.OAEP(
+            mgf=padding.MGF1(algorithm=hashes.SHA1()),
+            algorithm=hashes.SHA1(),
+            label=None
+        )
+    )
+    print(ciphertext.encode('base64'))
+
+if __name__ == '__main__':
+    main()

tests/fixtures/config/ansible/git/common-config/zuul.yaml

@@ -34,6 +34,21 @@
         verified: 0
     precedence: high
 
+- secret:
+    name: test_secret
+    data:
+      username: test-username
+      password: !encrypted/pkcs1 |
+        BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+        L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+        ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+        3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+        Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+        xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+        aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+        Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+        +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+
 - job:
     name: python27
     pre-run: pre

tests/unit/test_model.py

@@ -302,6 +302,29 @@ class TestJob(BaseTestCase):
         tenant = model.Tenant('tenant')
         layout = model.Layout()
 
+        conf = yaml.safe_load('''
+- secret:
+    name: pypi-credentials
+    data:
+      username: test-username
+      password: !encrypted/pkcs1 |
+        BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+        L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+        ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+        3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+        Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+        xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+        aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+        Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+        +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+''')[0]['secret']
+
+        conf['_source_context'] = self.context
+        conf['_start_mark'] = self.start_mark
+
+        secret = configloader.SecretParser.fromYaml(layout, conf)
+        layout.addSecret(secret)
+
         base = configloader.JobParser.fromYaml(tenant, layout, {
             '_source_context': self.context,
             '_start_mark': self.start_mark,

zuul/configloader.py

@@ -87,7 +87,7 @@ def configuration_exceptions(stanza, conf):
 
 
 class ZuulSafeLoader(yaml.SafeLoader):
-    zuul_node_types = frozenset(('job', 'nodeset', 'pipeline',
+    zuul_node_types = frozenset(('job', 'nodeset', 'secret', 'pipeline',
                                  'project', 'project-template'))
 
     def __init__(self, stream, context):
@@ -125,6 +125,18 @@ repo {repo} on branch {branch}.  The error was:
         loader.dispose()
 
 
+class EncryptedPKCS1(yaml.YAMLObject):
+    yaml_tag = u'!encrypted/pkcs1'
+    yaml_loader = yaml.SafeLoader
+
+    def __init__(self, ciphertext):
+        self.ciphertext = ciphertext
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        return cls(node.value)
+
+
 class NodeSetParser(object):
     @staticmethod
     def getSchema():
@@ -151,6 +163,28 @@ class NodeSetParser(object):
         return ns
 
 
+class SecretParser(object):
+    @staticmethod
+    def getSchema():
+        data = {str: vs.Any(str, EncryptedPKCS1)}
+
+        secret = {vs.Required('name'): str,
+                  vs.Required('data'): data,
+                  '_source_context': model.SourceContext,
+                  '_start_mark': yaml.Mark,
+                  }
+
+        return vs.Schema(secret)
+
+    @staticmethod
+    def fromYaml(layout, conf):
+        with configuration_exceptions('secret', conf):
+            SecretParser.getSchema()(conf)
+        s = model.Secret(conf['name'])
+        s.secret_data = conf['data']
+        return s
+
+
 class JobParser(object):
     @staticmethod
     def getSchema():
@@ -906,6 +940,9 @@ class TenantParser(object):
         for config_nodeset in data.nodesets:
             layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
 
+        for config_secret in data.secrets:
+            layout.addSecret(SecretParser.fromYaml(layout, config_secret))
+
         for config_job in data.jobs:
             layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
 

zuul/model.py

@@ -515,6 +515,35 @@ class NodeRequest(object):
         self.state_time = data['state_time']
 
 
+class Secret(object):
+    """A collection of private data.
+
+    In configuration, Secrets are collections of private data in
+    key-value pair format.  They are defined as top-level
+    configuration objects and then referenced by Jobs.
+
+    """
+
+    def __init__(self, name):
+        self.name = name
+        # The secret data may or may not be encrypted.  This attribute
+        # is named 'secret_data' to make it easy to search for and
+        # spot where it is directly used.
+        self.secret_data = {}
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __eq__(self, other):
+        if not isinstance(other, Secret):
+            return False
+        return (self.name == other.name and
+                self.secret_data == other.secret_data)
+
+    def __repr__(self):
+        return '<Secret %s>' % (self.name,)
+
+
 class SourceContext(object):
     """A reference to the branch of a project in configuration.
 
@@ -2132,6 +2161,7 @@ class UnparsedTenantConfig(object):
         self.project_templates = []
         self.projects = {}
         self.nodesets = []
+        self.secrets = []
 
     def copy(self):
         r = UnparsedTenantConfig()
@@ -2140,6 +2170,7 @@ class UnparsedTenantConfig(object):
         r.project_templates = copy.deepcopy(self.project_templates)
         r.projects = copy.deepcopy(self.projects)
         r.nodesets = copy.deepcopy(self.nodesets)
+        r.secrets = copy.deepcopy(self.secrets)
         return r
 
     def extend(self, conf):
@@ -2150,6 +2181,7 @@ class UnparsedTenantConfig(object):
             for k, v in conf.projects.items():
                 self.projects.setdefault(k, []).extend(v)
             self.nodesets.extend(conf.nodesets)
+            self.secrets.extend(conf.secrets)
             return
 
         if not isinstance(conf, list):
@@ -2178,6 +2210,8 @@ class UnparsedTenantConfig(object):
                 self.pipelines.append(value)
             elif key == 'nodeset':
                 self.nodesets.append(value)
+            elif key == 'secret':
+                self.secrets.append(value)
             else:
                 raise Exception("Configuration item `%s` not recognized "
                                 "(when parsing %s)" %
@@ -2200,6 +2234,7 @@ class Layout(object):
         # inherit from the reference definition.
         self.jobs = {'noop': [Job('noop')]}
         self.nodesets = {}
+        self.secrets = {}
 
     def getJob(self, name):
         if name in self.jobs:
@@ -2233,6 +2268,11 @@ class Layout(object):
             raise Exception("NodeSet %s already defined" % (nodeset.name,))
         self.nodesets[nodeset.name] = nodeset
 
+    def addSecret(self, secret):
+        if secret.name in self.secrets:
+            raise Exception("Secret %s already defined" % (secret.name,))
+        self.secrets[secret.name] = secret
+
     def addPipeline(self, pipeline):
         self.pipelines[pipeline.name] = pipeline