Ensure SY gets redacted rendered documents

- Introduced query parameter 'cleartext-secrets' yields
  cleartext secrets in encrypted documents if true, else
  the values are redacted.
- Introduced CLI flag '--cleartext-secrets' yields cleartext
  secrets in encrypted documents.

Change-Id: I8a950c8bded3550f4aab8e6dc662ea330c2fb73f

charts/shipyard/values.yaml

@@ -370,6 +370,7 @@ conf:
     workflow_orchestrator:get_configdocs: rule:admin_read_access
     workflow_orchestrator:commit_configdocs: rule:admin_create
     workflow_orchestrator:get_renderedconfigdocs: rule:admin_read_access
+    workflow_orchestrator:get_renderedconfigdocs_cleartext: rule:admin_read_access
     workflow_orchestrator:list_workflows: rule:admin_read_access
     workflow_orchestrator:get_workflow: rule:admin_read_access
     workflow_orchestrator:get_notedetails: rule:admin_read_access

doc/source/API.rst

@@ -259,8 +259,11 @@ Returns the full set of configdocs in their rendered form.
 
 Query Parameters
 ''''''''''''''''
-version=committed | last_site_action | successful_site_action | **buffer**
+- version=committed | last_site_action | successful_site_action | **buffer**
   Return the documents for the version specified - buffer by default.
+- cleartext-secrets=true/**false**
+  If true then returns cleartext secrets in encrypted documents, otherwise
+  those values are redacted.
 
 Responses
 '''''''''

doc/source/CLI.rst

@@ -726,6 +726,7 @@ applying Deckhand layering and substitution.
 
     shipyard get renderedconfigdocs
         [--committed | --last-site-action | --successful-site-action | --buffer]
+        [--cleartext-secrets]
 
     Example:
         shipyard get renderedconfigdocs
@@ -743,6 +744,10 @@ applying Deckhand layering and substitution.
   Retrieve the documents that have been loaded into Shipyard since the
   prior commit. (default)
 
+\--cleartext-secrets
+  Returns secrets as cleartext for encrypted documents if the user has the appropriate
+  permissions in the target environment.
+
 Sample
 ^^^^^^
 

src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py

@@ -98,7 +98,7 @@ class ConfigDocsResource(BaseResource):
         Returns a collection of documents
         """
         version = (req.params.get('version') or 'buffer')
-        cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
+        cleartext_secrets = req.get_param_as_bool('cleartext-secrets') or False
         self._validate_version_parameter(version)
         helper = ConfigdocsHelper(req.context)
         # Not reformatting to JSON or YAML since just passing through

src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/rendered_configdocs_api.py

@@ -43,11 +43,19 @@ class RenderedConfigDocsResource(BaseResource):
         Returns the whole set of rendered documents
         """
         version = (req.params.get('version') or 'buffer')
+        cleartext_secrets = req.get_param_as_bool('cleartext-secrets') or False
         self._validate_version_parameter(version)
         helper = ConfigdocsHelper(req.context)
+
+        # Check access to cleartext_secrets
+        if cleartext_secrets:
+            policy.check_auth(req.context,
+                              policy.GET_RENDEREDCONFIGDOCS_CLRTXT)
+
         resp.body = self.get_rendered_configdocs(
             helper=helper,
-            version=version
+            version=version,
+            cleartext_secrets=cleartext_secrets
         )
         resp.append_header('Content-Type', 'application/x-yaml')
         resp.status = falcon.HTTP_200
@@ -64,8 +72,9 @@ class RenderedConfigDocsResource(BaseResource):
                 retry=False,
             )
 
-    def get_rendered_configdocs(self, helper, version='buffer'):
+    def get_rendered_configdocs(self, helper, version='buffer',
+                                cleartext_secrets=False):
         """
         Get and return the rendered configdocs from the helper/Deckhand
         """
-        return helper.get_rendered_configdocs(version)
+        return helper.get_rendered_configdocs(version, cleartext_secrets)

src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py

@@ -375,7 +375,7 @@ class ConfigdocsHelper(object):
             status=falcon.HTTP_404,
             retry=False)
 
-    def get_rendered_configdocs(self, version=BUFFER):
+    def get_rendered_configdocs(self, version=BUFFER, cleartext_secrets=False):
         """
         Returns the rendered configuration documents for the specified
         revision (by name BUFFER, COMMITTED, LAST_SITE_ACTION,
@@ -397,6 +397,7 @@ class ConfigdocsHelper(object):
 
             try:
                 return self.deckhand.get_rendered_docs_from_revision(
+                    cleartext_secrets=cleartext_secrets,
                     revision_id=revision_id)
             except DeckhandError as de:
                 raise ApiError(

src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py

@@ -232,7 +232,8 @@ class DeckhandClient(object):
                                     response.text))})
         return errors
 
-    def get_rendered_docs_from_revision(self, revision_id, bucket_id=None):
+    def get_rendered_docs_from_revision(self, revision_id, bucket_id=None,
+                                        cleartext_secrets=False):
         """
         Returns the full set of rendered documents for a revision
         """
@@ -240,9 +241,11 @@ class DeckhandClient(object):
             DeckhandPaths.RENDERED_REVISION_DOCS
         ).format(revision_id)
 
-        query = None
+        query = {}
         if bucket_id is not None:
             query = {'status.bucket': bucket_id}
+        if cleartext_secrets is True:
+            query['cleartext-secrets'] = 'true'
         response = self._get_request(url, params=query)
         self._handle_bad_response(response)
         return response.text

src/bin/shipyard_airflow/shipyard_airflow/policy.py

@@ -38,6 +38,7 @@ CREATE_CONFIGDOCS = 'workflow_orchestrator:create_configdocs'
 GET_CONFIGDOCS = 'workflow_orchestrator:get_configdocs'
 COMMIT_CONFIGDOCS = 'workflow_orchestrator:commit_configdocs'
 GET_RENDEREDCONFIGDOCS = 'workflow_orchestrator:get_renderedconfigdocs'
+GET_RENDEREDCONFIGDOCS_CLRTXT = 'workflow_orchestrator:get_renderedconfigdocs_cleartext'  # noqa
 LIST_WORKFLOWS = 'workflow_orchestrator:list_workflows'
 GET_WORKFLOW = 'workflow_orchestrator:get_workflow'
 GET_NOTEDETAILS = 'workflow_orchestrator:get_notedetails'
@@ -187,6 +188,16 @@ class ShipyardPolicy(object):
                 'method': 'GET'
             }]
         ),
+        policy.DocumentedRuleDefault(
+            GET_RENDEREDCONFIGDOCS_CLRTXT,
+            RULE_ADMIN_REQUIRED,
+            ('Retrieve the configuration documents with cleartext secrets '
+             'rendered by Deckhand into a complete design'),
+            [{
+                'path': '/api/v1.0/renderedconfigdocs',
+                'method': 'GET'
+            }]
+        ),
         policy.DocumentedRuleDefault(
             LIST_WORKFLOWS,
             RULE_ADMIN_REQUIRED,

src/bin/shipyard_airflow/tests/unit/control/test_rendered_configdocs_api.py

@@ -52,7 +52,7 @@ def test_get_rendered_configdocs():
         helper = ConfigdocsHelper(CTX)
         rcdr.get_rendered_configdocs(helper, version='buffer')
 
-    mock_method.assert_called_once_with('buffer')
+    mock_method.assert_called_once_with('buffer', False)
 
 
 def test_get_rendered_last_site_action_configdocs():
@@ -68,7 +68,7 @@ def test_get_rendered_last_site_action_configdocs():
         helper = ConfigdocsHelper(CTX)
         rcdr.get_rendered_configdocs(helper, version='last_site_action')
 
-    mock_method.assert_called_once_with('last_site_action')
+    mock_method.assert_called_once_with('last_site_action', False)
 
 
 def test_get_rendered_successful_site_action_configdocs():
@@ -84,4 +84,4 @@ def test_get_rendered_successful_site_action_configdocs():
         helper = ConfigdocsHelper(CTX)
         rcdr.get_rendered_configdocs(helper, version='successful_site_action')
 
-    mock_method.assert_called_once_with('successful_site_action')
+    mock_method.assert_called_once_with('successful_site_action', False)

src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py

@@ -96,7 +96,7 @@ class ShipyardClient(BaseClient):
         url = ApiPaths.GET_CONFIGDOCS.value.format(self.get_endpoint())
         return self.get_resp(url, query_params)
 
-    def get_rendereddocs(self, version='buffer'):
+    def get_rendereddocs(self, version='buffer', cleartext_secrets=False):
         """
         :param str version: committed|buffer|last_site_action|
                             successful_site_action
@@ -104,6 +104,8 @@ class ShipyardClient(BaseClient):
         :rtype: Response object
         """
         query_params = {"version": version}
+        if cleartext_secrets is True:
+            query_params['cleartext-secrets'] = 'true'
         url = ApiPaths.GET_RENDERED.value.format(
             self.get_endpoint()
         )

src/bin/shipyard_client/shipyard_client/cli/get/actions.py

@@ -119,16 +119,18 @@ class GetConfigdocsStatus(CliAction):
 class GetRenderedConfigdocs(CliAction):
     """Action to Get Rendered Configdocs"""
 
-    def __init__(self, ctx, version):
+    def __init__(self, ctx, version, cleartext_secrets=False):
         """Sets parameters."""
         super().__init__(ctx)
         self.logger.debug("GetRenderedConfigdocs action initialized")
         self.version = version
+        self.cleartext_secrets = cleartext_secrets
 
     def invoke(self):
         """Calls API Client and formats response from API Client"""
         self.logger.debug("Calling API Client get_rendereddocs.")
-        return self.get_api_client().get_rendereddocs(version=self.version)
+        return self.get_api_client().get_rendereddocs(
+            version=self.version, cleartext_secrets=self.cleartext_secrets)
 
     # Handle 404 with default error handler for cli.
     cli_handled_err_resp_codes = [404]

src/bin/shipyard_client/shipyard_client/cli/get/commands.py

@@ -100,7 +100,6 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either "
     'executed site action.')
 @click.option(
     '--cleartext-secrets',
-    '-t',
     help='Returns cleartext secrets in documents',
     is_flag=True)
 @click.pass_context
@@ -170,14 +169,19 @@ SHORT_DESC_RENDEREDCONFIGDOCS = (
     flag_value='successful_site_action',
     help='Holds the revision information for the most recent successfully '
     'executed site action.')
+@click.option(
+    '--cleartext-secrets',
+    help='Returns cleartext secrets in encrypted documents',
+    is_flag=True)
 @click.pass_context
 def get_renderedconfigdocs(ctx, buffer, committed, last_site_action,
-                           successful_site_action):
+                           successful_site_action, cleartext_secrets):
     # Get version
     _version = get_version(ctx, buffer, committed, last_site_action,
                            successful_site_action)
 
-    click.echo(GetRenderedConfigdocs(ctx, _version).invoke_and_return_resp())
+    click.echo(GetRenderedConfigdocs(ctx, _version,
+                                     cleartext_secrets).invoke_and_return_resp())
 
 
 DESC_WORKFLOWS = """

src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py

@@ -88,7 +88,7 @@ def test_get_renderedconfigdocs(*args):
     runner = CliRunner()
     with patch.object(GetRenderedConfigdocs, '__init__') as mock_method:
         runner.invoke(shipyard, [auth_vars, 'get', 'renderedconfigdocs'])
-    mock_method.assert_called_once_with(ANY, 'buffer')
+    mock_method.assert_called_once_with(ANY, 'buffer', False)
 
 
 def test_get_renderedconfigdocs_negative(*args):