Support complex data structures as secrets

This allows us to encode complex data structures, such as entire
openstack clouds.yaml cloud configs, as secrets.

Change-Id: I5fbafa41b9ab180842a4dd7bd82e2603f38e6644

doc/source/user/config.rst

@@ -1332,9 +1332,9 @@ branch will not immediately produce a configuration error.
       :required:
 
       A dictionary which will be added to the Ansible variables
-      available to the job.  The values can either be plain text
-      strings, or encrypted values.  See :ref:`encryption` for more
-      information.
+      available to the job.  The values can be any of the normal YAML
+      data types (strings, integers, dictionaries or lists) or
+      encrypted strings.  See :ref:`encryption` for more information.
 
 .. _nodeset:
 

tests/fixtures/config/secrets/git/org_project2/zuul-complex.yaml

@@ -0,0 +1,44 @@
+- secret:
+    name: complex_secret
+    data:
+      dict:
+        username: test-username
+        password: !encrypted/pkcs1-oaep |
+          BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+          L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+          ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+          3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+          Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+          xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+          aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+          Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+          +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+      profile: cloudy
+      list:
+        - one
+        - !encrypted/pkcs1-oaep |
+            BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+            L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
+            ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
+            3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
+            Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
+            xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
+            aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
+            Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
+            +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+        - three
+
+- job:
+    parent: base
+    name: project2-complex
+    run: playbooks/secret.yaml
+    secrets:
+      - complex_secret
+
+- project:
+    check:
+      jobs:
+        - project2-complex
+    gate:
+      jobs:
+        - noop

tests/unit/test_v3.py

@@ -3474,6 +3474,32 @@ class TestSecrets(ZuulTestCase):
         self.assertIn('already defined in project org/project1',
                       A.messages[0])
 
+    def test_complex_secret(self):
+        # Test that we can use a complex secret
+        with open(os.path.join(FIXTURE_DIR,
+                               'config/secrets/git/',
+                               'org_project2/zuul-complex.yaml')) as f:
+            config = f.read()
+
+        file_dict = {'zuul.yaml': config}
+        A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 1, "A should report success")
+        self.assertHistory([
+            dict(name='project2-complex', result='SUCCESS', changes='1,1'),
+        ])
+        secret = {'complex_secret':
+                  {'dict': {'password': 'test-password',
+                            'username': 'test-username'},
+                   'list': ['one', 'test-password', 'three'],
+                   'profile': 'cloudy'}}
+
+        self.assertEqual(
+            self._getSecrets('project2-complex', 'playbooks'),
+            [secret])
+
 
 class TestSecretInheritance(ZuulTestCase):
     tenant_config_file = 'config/secret-inheritance/main.yaml'

zuul/configloader.py

@@ -470,10 +470,8 @@ class SecretParser(object):
         self.schema = self.getSchema()
 
     def getSchema(self):
-        data = {str: vs.Any(str, EncryptedPKCS1_OAEP)}
-
         secret = {vs.Required('name'): str,
-                  vs.Required('data'): data,
+                  vs.Required('data'): dict,
                   '_source_context': model.SourceContext,
                   '_start_mark': ZuulMark,
                   }

zuul/model.py

@@ -754,18 +754,31 @@ class Secret(ConfigObject):
     def __repr__(self):
         return '<Secret %s>' % (self.name,)
 
+    def _decrypt(self, private_key, secret_data):
+        # recursive function to decrypt data
+        if hasattr(secret_data, 'decrypt'):
+            return secret_data.decrypt(private_key)
+
+        if isinstance(secret_data, (dict, types.MappingProxyType)):
+            decrypted_secret_data = {}
+            for k, v in secret_data.items():
+                decrypted_secret_data[k] = self._decrypt(private_key, v)
+            return decrypted_secret_data
+
+        if isinstance(secret_data, (list, tuple)):
+            decrypted_secret_data = []
+            for v in secret_data:
+                decrypted_secret_data.append(self._decrypt(private_key, v))
+            return decrypted_secret_data
+
+        return secret_data
+
     def decrypt(self, private_key):
         """Return a copy of this secret with any encrypted data decrypted.
         Note that the original remains encrypted."""
 
         r = Secret(self.name, self.source_context)
-        decrypted_secret_data = {}
-        for k, v in self.secret_data.items():
-            if hasattr(v, 'decrypt'):
-                decrypted_secret_data[k] = v.decrypt(private_key)
-            else:
-                decrypted_secret_data[k] = v
-        r.secret_data = decrypted_secret_data
+        r.secret_data = self._decrypt(private_key, self.secret_data)
         return r