Browse Source

CLI: Add command to generate genesis bundle

Added a pegleg cli command to build genesis.sh bundle for
a site deployment.
Pegleg imports promenade engine, and uses promenade to build
and encrypt the genesis.sh deployment bundle.

Change-Id: I1a489459b2c56b7b53018c32aab5e6550c69e1d2
changes/92/613392/42
Ahmad Mahmoudi 7 months ago
parent
commit
c4f25b4d4f

+ 40
- 3
doc/source/cli/cli.rst View File

@@ -612,6 +612,42 @@ Example:
612 612
       secrets decrypt site1 -f \
613 613
       /opt/security-manifests/site/site1/passwords/password1.yaml
614 614
 
615
+genesis_bundle
616
+--------------
617
+
618
+Constructs genesis bundle based on a site configuration.
619
+
620
+.. note::
621
+  This command requires the environment variable PEGLEG_PASSPHRASE
622
+  to be set and at least 24 characters long, to be used for encrypting
623
+  genesis bundle data. PEGLEG_SALT must be set as well. There are no
624
+  constraints on its length, but at least 24 characters is recommended.
625
+
626
+
627
+**-b / --build-dir** (Required).
628
+
629
+Destination directory for the genesis bundle.
630
+
631
+**--include-validators** (Optional). False by default.
632
+
633
+A flag to request build genesis validation scripts as well.
634
+
635
+Usage:
636
+
637
+::
638
+    ./pegleg.sh site <options> genesis_bundle <site_name> \
639
+      -b <build_locaton> -k <encryption_passphrase/key> --validators
640
+
641
+Examples
642
+^^^^^^^^
643
+
644
+::
645
+
646
+    ./pegleg.sh site -r  ./site-manifests \
647
+      genesis_bundle site1 \
648
+      -b ../../site1_build \
649
+      -k yourEncryptionPassphrase \
650
+      --validators
615 651
 
616 652
 generate
617 653
 ^^^^^^^^
@@ -803,8 +839,9 @@ Where mandatory encrypted schema type is one of:
803 839
 P002 - Deckhand rendering is expected to complete without errors.
804 840
 P003 - All repos contain expected directories.
805 841
 
806
-.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/rendering.html
807
-.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/validation.html
842
+
843
+.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html
844
+.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation
808 845
 .. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument
809 846
 .. _Shipyard: https://github.com/openstack/airship-shipyard
810 847
 .. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables
@@ -878,4 +915,4 @@ Example with length specified:
878 915
 
879 916
 ::
880 917
 
881
-    ./pegleg.sh generate salt -l <length>
918
+    ./pegleg.sh generate salt -l <length>

+ 9
- 0
doc/source/exceptions.rst View File

@@ -68,10 +68,19 @@ PKI Exceptions
68 68
 --------------
69 69
 
70 70
 .. autoexception:: pegleg.engine.exceptions.IncompletePKIPairError
71
+
72
+Genesis Bundle Exceptions
73
+-------------------------
74
+
75
+.. autoexception:: pegleg.engine.exceptions.GenesisBundleEncryptionException
71 76
    :members:
72 77
    :show-inheritance:
73 78
    :undoc-members:
74 79
 
80
+.. autoexception:: pegleg.engine.exceptions.GenesisBundleGenerateException
81
+   :members:
82
+   :show-inheritance:
83
+
75 84
 Passphrase Exceptions
76 85
 ---------------------
77 86
 

+ 51
- 0
pegleg/cli.py View File

@@ -14,13 +14,16 @@
14 14
 
15 15
 import functools
16 16
 import logging
17
+import os
17 18
 import sys
18 19
 
19 20
 import click
20 21
 
21 22
 from pegleg import config
22 23
 from pegleg import engine
24
+from pegleg.engine import bundle
23 25
 from pegleg.engine import catalog
26
+from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
24 27
 from pegleg.engine.util.shipyard_helper import ShipyardHelper
25 28
 
26 29
 LOG = logging.getLogger(__name__)
@@ -412,6 +415,54 @@ def generate_pki(site_name, author):
412 415
     click.echo("Generated PKI files written to:\n%s" % '\n'.join(output_paths))
413 416
 
414 417
 
418
+@site.command(
419
+    'genesis_bundle',
420
+    help='Construct the genesis deployment bundle.')
421
+@click.option(
422
+    '-b',
423
+    '--build-dir',
424
+    'build_dir',
425
+    type=click.Path(file_okay=False, dir_okay=True, resolve_path=True),
426
+    required=True,
427
+    help='Destination directory to store the genesis bundle.')
428
+@click.option(
429
+    '--include-validators',
430
+    'validators',
431
+    is_flag=True,
432
+    default=False,
433
+    help='A flag to request generate genesis validation scripts in addition '
434
+         'to genesis.sh script.')
435
+@SITE_REPOSITORY_ARGUMENT
436
+def genesis_bundle(*, build_dir, validators, site_name):
437
+    prom_encryption_key = os.environ.get("PROMENADE_ENCRYPTION_KEY")
438
+    peg_encryption_key = os.environ.get("PEGLEG_PASSPHRASE")
439
+    encryption_key = None
440
+    if (prom_encryption_key and len(prom_encryption_key) > 24 and
441
+            peg_encryption_key and len(peg_encryption_key) > 24):
442
+        click.echo("WARNING: PROMENADE_ENCRYPTION_KEY is deprecated, "
443
+                   "using PEGLEG_PASSPHRASE instead", err=True)
444
+        config.set_passphrase(peg_encryption_key)
445
+        encryption_key = peg_encryption_key
446
+    elif prom_encryption_key and len(prom_encryption_key) > 24:
447
+        click.echo("ERROR: PROMENADE_ENCRYPTION_KEY is deprecated, "
448
+                   "use PEGLEG_PASSPHRASE instead", err=True)
449
+        raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set "
450
+                                   "and at least 24 characters long.")
451
+    elif peg_encryption_key and len(peg_encryption_key) > 24:
452
+        config.set_passphrase(peg_encryption_key)
453
+        encryption_key = peg_encryption_key
454
+    else:
455
+        raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set "
456
+                                   "and at least 24 characters long.")
457
+
458
+    PeglegSecretManagement.check_environment()
459
+    bundle.build_genesis(build_dir,
460
+                         encryption_key,
461
+                         validators,
462
+                         logging.DEBUG == LOG.getEffectiveLevel(),
463
+                         site_name)
464
+
465
+
415 466
 @main.group(help='Commands related to types')
416 467
 @MAIN_REPOSITORY_OPTION
417 468
 @REPOSITORY_CLONE_PATH_OPTION

+ 23
- 1
pegleg/config.py View File

@@ -26,7 +26,9 @@ except NameError:
26 26
         'clone_path': None,
27 27
         'site_path': 'site',
28 28
         'site_rev': None,
29
-        'type_path': 'type'
29
+        'type_path': 'type',
30
+        'passphrase': None,
31
+        'salt': None
30 32
     }
31 33
 
32 34
 
@@ -147,3 +149,23 @@ def set_rel_type_path(p):
147 149
     """Set the relative type path name."""
148 150
     p = p or 'type'
149 151
     GLOBAL_CONTEXT['type_path'] = p
152
+
153
+
154
+def set_passphrase(p):
155
+    """Set the passphrase for encryption and decryption."""
156
+    GLOBAL_CONTEXT['passphrase'] = p
157
+
158
+
159
+def get_passphrase():
160
+    """Get the passphrase for encryption and decryption."""
161
+    return GLOBAL_CONTEXT['passphrase']
162
+
163
+
164
+def set_salt(p):
165
+    """Set the salt for encryption and decryption."""
166
+    GLOBAL_CONTEXT['salt'] = p
167
+
168
+
169
+def get_salt():
170
+    """Get the salt for encryption and decryption."""
171
+    return GLOBAL_CONTEXT['salt']

+ 92
- 0
pegleg/engine/bundle.py View File

@@ -0,0 +1,92 @@
1
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#     http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+import logging
16
+import os
17
+import stat
18
+
19
+import click
20
+
21
+from pegleg.engine.exceptions import GenesisBundleEncryptionException
22
+from pegleg.engine.exceptions import GenesisBundleGenerateException
23
+from pegleg.engine import util
24
+from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
25
+
26
+from promenade.builder import Builder
27
+from promenade.config import Configuration
28
+from promenade import exceptions
29
+
30
+LOG = logging.getLogger(__name__)
31
+
32
+__all__ = [
33
+    'build_genesis',
34
+]
35
+
36
+
37
+def build_genesis(build_path, encryption_key, validators, debug, site_name):
38
+    """
39
+    Build the genesis deployment bundle, and store it in ``build_path``.
40
+
41
+    Build the genesis.sh script, base65-encode, encrypt and embed the
42
+    site configuration source documents in genesis.sh script.
43
+    If ``validators`` flag should be True, build the bundle validator
44
+    scripts as well.
45
+    Store the built deployment bundle in `build_path`.
46
+
47
+    :param str build_path: Directory path of the built genesis deployment
48
+    bundle
49
+    :param str encryption_key: Key to use to encrypt the bundled site
50
+    configuration in genesis.sh script.
51
+    :param bool validators: Whether to generate validator scripts
52
+    :param int debug: pegleg debug level to pass to promenade engine
53
+    for logging.
54
+    :return: None
55
+    """
56
+
57
+    # Raise an error if the build path exists. We don't want to overwrite it.
58
+    if os.path.isdir(build_path):
59
+        raise click.ClickException(
60
+            "{} already exists, remove it or specify a new "
61
+            "directory.".format(build_path))
62
+    # Get the list of config files
63
+    LOG.info('=== Building bootstrap scripts ===')
64
+
65
+    # Copy the site config, and site secrets to build directory
66
+    os.mkdir(build_path)
67
+    os.chmod(build_path, os.stat(build_path).st_mode | stat.S_IRWXU |
68
+             stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH)
69
+    documents = util.definition.documents_for_site(site_name)
70
+    secret_manager = PeglegSecretManagement(docs=documents)
71
+    documents = secret_manager.get_decrypted_secrets()
72
+    try:
73
+        # Use the promenade engine to build and encrypt the genesis bundle
74
+        c = Configuration(
75
+            documents=documents,
76
+            debug=debug,
77
+            substitute=True,
78
+            allow_missing_substitutions=False,
79
+            leave_kubectl=False)
80
+        if c.get_path('EncryptionPolicy:scripts.genesis') and encryption_key:
81
+            os.environ['PROMENADE_ENCRYPTION_KEY'] = encryption_key
82
+            os.environ['PEGLEG_PASSPHRASE'] = encryption_key
83
+            Builder(c, validators=validators).build_all(output_dir=build_path)
84
+        else:
85
+            raise GenesisBundleEncryptionException()
86
+
87
+    except exceptions.PromenadeException as e:
88
+        LOG.error('Build genesis bundle failed! {}.'.format(
89
+            e.display(debug=debug)))
90
+        raise GenesisBundleGenerateException()
91
+
92
+    LOG.info('=== Done! ===')

+ 16
- 0
pegleg/engine/exceptions.py View File

@@ -86,3 +86,19 @@ class PassphraseCatalogNotFoundException(PeglegBaseException):
86 86
     """Failed to find Catalog for Passphrases generation."""
87 87
     message = ('Could not find the Passphrase Catalog to generate '
88 88
                'the site Passphrases!')
89
+
90
+
91
+class GenesisBundleEncryptionException(PeglegBaseException):
92
+    """Exception raised when encryption of the genesis bundle fails."""
93
+
94
+    message = 'Encryption is required for genesis bundle, but no encryption ' \
95
+              'policy or key is specified.'
96
+
97
+
98
+class GenesisBundleGenerateException(PeglegBaseException):
99
+    """
100
+    Exception raised when pormenade engine fails to build the genesis
101
+    bundle.
102
+    """
103
+
104
+    message = 'Bundle generation failed on deckhand validation.'

+ 16
- 5
pegleg/engine/util/pegleg_secret_management.py View File

@@ -19,6 +19,7 @@ import re
19 19
 import click
20 20
 import yaml
21 21
 
22
+from pegleg import config
22 23
 from pegleg.engine.util.encryption import decrypt
23 24
 from pegleg.engine.util.encryption import encrypt
24 25
 from pegleg.engine.util import files
@@ -47,10 +48,11 @@ class PeglegSecretManagement(object):
47 48
             raise ValueError('Either `file_path` or `docs` must be '
48 49
                              'specified.')
49 50
 
50
-        if generated and not (author and catalog):
51
+        if generated and not (catalog and author):
51 52
             raise ValueError("If the document is generated, author and "
52 53
                              "catalog must be specified.")
53
-        self.__check_environment()
54
+
55
+        self.check_environment()
54 56
         self.file_path = file_path
55 57
         self.documents = list()
56 58
         self._generated = generated
@@ -68,8 +70,17 @@ class PeglegSecretManagement(object):
68 70
 
69 71
         self._author = author
70 72
 
71
-        self.passphrase = os.environ.get(ENV_PASSPHRASE).encode()
72
-        self.salt = os.environ.get(ENV_SALT).encode()
73
+        if config.get_passphrase() and config.get_salt():
74
+            self.passphrase = config.get_passphrase()
75
+            self.salt = config.get_salt()
76
+        elif config.get_passphrase() or config.get_salt():
77
+            raise ValueError("ERROR: Pegleg configuration must either have "
78
+                             "both a passphrase and a salt or neither.")
79
+        else:
80
+            self.passphrase = os.environ.get(ENV_PASSPHRASE).encode()
81
+            self.salt = os.environ.get(ENV_SALT).encode()
82
+            config.set_passphrase(self.passphrase)
83
+            config.set_salt(self.salt)
73 84
 
74 85
     def __iter__(self):
75 86
         """
@@ -79,7 +90,7 @@ class PeglegSecretManagement(object):
79 90
         return (doc.pegleg_document for doc in self.documents)
80 91
 
81 92
     @staticmethod
82
-    def __check_environment():
93
+    def check_environment():
83 94
         """
84 95
         Validate required environment variables for encryption or decryption.
85 96
 

+ 1
- 0
requirements.txt View File

@@ -9,3 +9,4 @@ python-dateutil==2.7.3
9 9
 rstr==2.2.6
10 10
 git+https://github.com/openstack/airship-deckhand.git@49ad9f38842f7f1ecb86d907d86d332f8186eb8c
11 11
 git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client
12
+git+https://github.com/openstack/airship-promenade.git@a6e8fdbe22bd153c78a008b92cd5d1c245bc63e3

+ 144
- 0
tests/unit/engine/test_build_genesis_bundle.py View File

@@ -0,0 +1,144 @@
1
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#     http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+import logging
16
+import os
17
+
18
+import mock
19
+import pytest
20
+import yaml
21
+
22
+from pegleg import config
23
+from pegleg.engine import bundle
24
+from pegleg.engine.exceptions import GenesisBundleEncryptionException
25
+from pegleg.engine.exceptions import GenesisBundleGenerateException
26
+from pegleg.engine.util import files
27
+from pegleg.engine.util.pegleg_secret_management import ENV_PASSPHRASE
28
+from pegleg.engine.util.pegleg_secret_management import ENV_SALT
29
+
30
+from tests.unit.fixtures import temp_path
31
+
32
+SITE_DEFINITION = """
33
+---
34
+# High-level pegleg site definition file
35
+schema: pegleg/SiteDefinition/v1
36
+metadata:
37
+  schema: metadata/Document/v1
38
+  layeringDefinition:
39
+    abstract: false
40
+    layer: site
41
+  # NEWSITE-CHANGEME: Replace with the site name
42
+  name: test_site
43
+  storagePolicy: cleartext
44
+data:
45
+  # The type layer this site will delpoy with. Type layer is found in the
46
+  # type folder.
47
+  site_type: foundry
48
+...
49
+
50
+"""
51
+
52
+SITE_CONFIG_DATA = """
53
+---
54
+schema: promenade/EncryptionPolicy/v1
55
+metadata:
56
+  schema: metadata/Document/v1
57
+  name: encryption-policy
58
+  layeringDefinition:
59
+    abstract: false
60
+    layer: site
61
+  storagePolicy: cleartext
62
+data:
63
+  scripts:
64
+    genesis:
65
+      gpg: {}
66
+    join:
67
+      gpg: {}
68
+---
69
+schema: deckhand/LayeringPolicy/v1
70
+metadata:
71
+  schema: metadata/Control/v1
72
+  name: layering-policy
73
+data:
74
+  layerOrder:
75
+    - global
76
+    - type
77
+    - site
78
+---
79
+schema: deckhand/Passphrase/v1
80
+metadata:
81
+  schema: metadata/Document/v1
82
+  name: ceph_swift_keystone_password
83
+  layeringDefinition:
84
+    abstract: false
85
+    layer: site
86
+  storagePolicy: cleartext
87
+data: ABAgagajajkb839215387
88
+...
89
+"""
90
+
91
+
92
+@mock.patch.dict(os.environ, {
93
+    ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
94
+    ENV_SALT: 'MySecretSalt'
95
+})
96
+def test_no_encryption_key(temp_path):
97
+    # Write the test data to temp file
98
+    config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
99
+    base_config_dir = os.path.join(temp_path, 'config_dir')
100
+    config.set_site_repo(base_config_dir)
101
+    config_dir = os.path.join(base_config_dir, 'site', 'test_site')
102
+
103
+    config_path = os.path.join(config_dir, 'config_file.yaml')
104
+    build_dir = os.path.join(temp_path, 'build_dir')
105
+    os.makedirs(config_dir)
106
+
107
+    files.write(config_path, config_data)
108
+    files.write(os.path.join(config_dir, "site-definition.yaml"),
109
+                yaml.safe_load_all(SITE_DEFINITION))
110
+
111
+    with pytest.raises(GenesisBundleEncryptionException,
112
+                       match=r'.*no encryption policy or key is specified.*'):
113
+        bundle.build_genesis(build_path=build_dir,
114
+                             encryption_key=None,
115
+                             validators=False,
116
+                             debug=logging.ERROR,
117
+                             site_name="test_site")
118
+
119
+
120
+@mock.patch.dict(os.environ, {
121
+    ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
122
+    ENV_SALT: 'MySecretSalt'
123
+})
124
+def test_failed_deckhand_validation(temp_path):
125
+    # Write the test data to temp file
126
+    config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
127
+    base_config_dir = os.path.join(temp_path, 'config_dir')
128
+    config.set_site_repo(base_config_dir)
129
+    config_dir = os.path.join(base_config_dir, 'site', 'test_site')
130
+
131
+    config_path = os.path.join(config_dir, 'config_file.yaml')
132
+    build_dir = os.path.join(temp_path, 'build_dir')
133
+    os.makedirs(config_dir)
134
+    files.write(config_path, config_data)
135
+    files.write(os.path.join(config_dir, "site-definition.yaml"),
136
+                yaml.safe_load_all(SITE_DEFINITION))
137
+    key = 'MyverYSecretEncryptionKey382803'
138
+    with pytest.raises(GenesisBundleGenerateException,
139
+                       match=r'.*failed on deckhand validation.*'):
140
+        bundle.build_genesis(build_path=build_dir,
141
+                             encryption_key=key,
142
+                             validators=False,
143
+                             debug=logging.ERROR,
144
+                             site_name="test_site")

Loading…
Cancel
Save