Expects Redacted Raw Documents

- Adds a query parameter 'cleartext-secrets' to get full raw documents.
- Adds CLI flag to get full raw documents.

Change-Id: If38974c8433c8360cc47ae1273720ad76e87a6fd

doc/source/API.rst

@@ -220,8 +220,11 @@ Returns the source documents for a collection of documents
 
 Query Parameters
 ''''''''''''''''
-version=committed | last_site_action | successful_site_action | **buffer**
+- version=committed | last_site_action | successful_site_action | **buffer**
   Return the documents for the version specified - buffer by default.
+- cleartext-secrets=true/**false**
+  If true then returns cleartext secrets in encrypted documents, otherwise
+  those values are redacted.
 
 Responses
 '''''''''

doc/source/CLI.rst

@@ -647,6 +647,7 @@ differences between the 'committed' and 'buffer' revision (default behavior).
     shipyard get configdocs
         [--collection=<collection>]
         [--committed | --last-site-action | --successful-site-action | --buffer]
+        [--cleartext-secrets]
 
     Example:
         shipyard get configdocs --collection=design
@@ -675,6 +676,10 @@ differences between the 'committed' and 'buffer' revision (default behavior).
   prior commit. If no documents have been loaded into the buffer for this
   collection, this will return an empty response (default)
 
+\--cleartext-secrets
+  Returns cleartext secrets in encrypted documents, otherwise those values
+  are redacted. Only impacts returned documents, not lists of documents.
+
 Sample
 ^^^^^^
 

src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py

@@ -98,11 +98,13 @@ class ConfigDocsResource(BaseResource):
         Returns a collection of documents
         """
         version = (req.params.get('version') or 'buffer')
+        cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
         self._validate_version_parameter(version)
         helper = ConfigdocsHelper(req.context)
         # Not reformatting to JSON or YAML since just passing through
         resp.body = self.get_collection(
-            helper=helper, collection_id=collection_id, version=version)
+            helper=helper, collection_id=collection_id, version=version,
+            cleartext_secrets=cleartext_secrets)
         resp.append_header('Content-Type', 'application/x-yaml')
         resp.status = falcon.HTTP_200
 
@@ -116,13 +118,15 @@ class ConfigDocsResource(BaseResource):
                 status=falcon.HTTP_400,
                 retry=False, )
 
-    def get_collection(self, helper, collection_id, version='buffer'):
+    def get_collection(self, helper, collection_id, version='buffer',
+                       cleartext_secrets=False):
         """
         Attempts to retrieve the specified collection of documents
         either from the buffer, committed version, last site action
         or successful site action, as specified
         """
-        return helper.get_collection_docs(version, collection_id)
+        return helper.get_collection_docs(version, collection_id,
+                                          cleartext_secrets)
 
     def post_collection(self,
                         helper,

src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py

@@ -318,18 +318,21 @@ class ConfigdocsHelper(object):
         rev = self._get_revision_dict().get(target_revision)
         return rev['id'] if rev else None
 
-    def get_collection_docs(self, version, collection_id):
+    def get_collection_docs(self, version, collection_id,
+                            cleartext_secrets=False):
         """
         Returns the requested collection of docs based on the version
         specifier. The default is set as buffer.
         """
         LOG.info('Retrieving collection %s from %s', collection_id, version)
         if version in [COMMITTED, LAST_SITE_ACTION, SUCCESSFUL_SITE_ACTION]:
-            return self._get_target_docs(collection_id, version)
+            return self._get_target_docs(collection_id, version,
+                                         cleartext_secrets)
 
-        return self._get_doc_from_buffer(collection_id)
+        return self._get_doc_from_buffer(collection_id,
+                                         cleartext_secrets=cleartext_secrets)
 
-    def _get_doc_from_buffer(self, collection_id):
+    def _get_doc_from_buffer(self, collection_id, cleartext_secrets=False):
         """
         Returns the collection if it exists in the buffer.
         If the buffer contains the collection, the latest
@@ -343,7 +346,8 @@ class ConfigdocsHelper(object):
             # revision exists
             buffer_id = self.get_revision_id(BUFFER)
             return self.deckhand.get_docs_from_revision(
-                revision_id=buffer_id, bucket_id=collection_id)
+                revision_id=buffer_id, bucket_id=collection_id,
+                cleartext_secrets=cleartext_secrets)
         raise ApiError(
             title='No documents to retrieve',
             description=('The Shipyard buffer is empty or does not contain '
@@ -351,7 +355,8 @@ class ConfigdocsHelper(object):
             status=falcon.HTTP_404,
             retry=False)
 
-    def _get_target_docs(self, collection_id, target_rev):
+    def _get_target_docs(self, collection_id, target_rev,
+                         cleartext_secrets=False):
         """
         Returns the collection if it exists as committed, last_site_action
         or successful_site_action.
@@ -360,7 +365,8 @@ class ConfigdocsHelper(object):
 
         if revision_id:
             return self.deckhand.get_docs_from_revision(
-                revision_id=revision_id, bucket_id=collection_id)
+                revision_id=revision_id, bucket_id=collection_id,
+                cleartext_secrets=cleartext_secrets)
 
         raise ApiError(
             title='No documents to retrieve',

src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py

@@ -184,20 +184,25 @@ class DeckhandClient(object):
         diff = yaml.safe_load(response.text)
         return diff
 
-    def get_docs_from_revision(self, revision_id, bucket_id=None):
+    def get_docs_from_revision(self, revision_id, bucket_id=None,
+                               cleartext_secrets=False):
         """
         Retrieves the collection of docs from the revision specified
         for the bucket_id specified
+        cleartext_secrets: Should deckhand show or redact secrets.
         :returns: a string representing the response.text from Deckhand
         """
+
         url = DeckhandClient.get_path(
             DeckhandPaths.REVISION_DOCS
         ).format(revision_id)
 
         # if a bucket_id is specified, limit the response to a bucket
-        query = None
+        query = {}
         if bucket_id is not None:
             query = {'status.bucket': bucket_id}
+        if cleartext_secrets is True:
+            query['cleartext-secrets'] = 'true'
         response = self._get_request(url, params=query)
         self._handle_bad_response(response)
         return response.text
@@ -336,6 +341,9 @@ class DeckhandClient(object):
                 'X-Auth-Token': get_token()
             }
 
+            if not params:
+                params = None
+
             DeckhandClient._log_request('GET', url, params)
             response = requests.get(
                 url,

src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py

@@ -96,7 +96,7 @@ class TestConfigDocsResource():
             helper = ConfigdocsHelper(CTX)
             cdr.get_collection(helper, 'apples')
 
-        mock_method.assert_called_once_with('buffer', 'apples')
+        mock_method.assert_called_once_with('buffer', 'apples', False)
 
     @patch.object(ConfigdocsHelper, 'is_collection_in_buffer',
                   lambda x, y: True)

src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py

@@ -503,7 +503,7 @@ def test_get_collection_docs():
     """
     helper = ConfigdocsHelper(CTX)
     helper.deckhand.get_docs_from_revision = (
-        lambda revision_id, bucket_id: "{'yaml': 'yaml'}")
+        lambda revision_id, bucket_id, cleartext_secrets: "{'yaml': 'yaml'}")
     helper._get_revision_dict = lambda: REV_EMPTY_DICT
     helper.deckhand.get_diff = (
         lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT)

src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py

@@ -68,7 +68,8 @@ class ShipyardClient(BaseClient):
         )
         return self.post_resp(url, query_params, document_data)
 
-    def get_configdocs(self, collection_id=None, version='buffer'):
+    def get_configdocs(self, collection_id=None, version='buffer',
+                       cleartext_secrets=False):
         """
         Get the collection of documents from deckhand specified by
         collection id
@@ -78,6 +79,8 @@ class ShipyardClient(BaseClient):
         :rtype: Response object
         """
         query_params = {"version": version}
+        if cleartext_secrets is True:
+            query_params['cleartext-secrets'] = 'true'
         url = ApiPaths.POST_GET_CONFIG.value.format(
             self.get_endpoint(),
             collection_id)

src/bin/shipyard_client/shipyard_client/cli/get/actions.py

@@ -48,7 +48,7 @@ class GetActions(CliAction):
 class GetConfigdocs(CliAction):
     """Action to Get Configdocs"""
 
-    def __init__(self, ctx, collection, version):
+    def __init__(self, ctx, collection, version, cleartext_secrets=False):
         """Sets parameters."""
         super().__init__(ctx)
         self.logger.debug(
@@ -56,12 +56,14 @@ class GetConfigdocs(CliAction):
             "version=%s" % (collection, version))
         self.collection = collection
         self.version = version
+        self.cleartext_secrets = cleartext_secrets
 
     def invoke(self):
         """Calls API Client and formats response from API Client"""
         self.logger.debug("Calling API Client get_configdocs.")
         return self.get_api_client().get_configdocs(
-            collection_id=self.collection, version=self.version)
+            collection_id=self.collection, version=self.version,
+            cleartext_secrets=self.cleartext_secrets)
 
     # Handle 404 with default error handler for cli.
     cli_handled_err_resp_codes = [404]

src/bin/shipyard_client/shipyard_client/cli/get/commands.py

@@ -98,16 +98,22 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either "
     flag_value='successful_site_action',
     help='Holds the revision information for the most recent successfully '
     'executed site action.')
+@click.option(
+    '--cleartext-secrets',
+    '-t',
+    help='Returns cleartext secrets in documents',
+    is_flag=True)
 @click.pass_context
 def get_configdocs(ctx, collection, buffer, committed, last_site_action,
-                   successful_site_action):
+                   successful_site_action, cleartext_secrets):
     if collection:
         # Get version
         _version = get_version(ctx, buffer, committed, last_site_action,
                                successful_site_action)
 
         click.echo(
-            GetConfigdocs(ctx, collection, _version).invoke_and_return_resp())
+            GetConfigdocs(ctx, collection, _version,
+                          cleartext_secrets).invoke_and_return_resp())
 
     else:
         compare_versions = check_reformat_versions(ctx,

src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py

@@ -58,7 +58,7 @@ def test_get_configdocs_with_passing_collection(*args):
     with patch.object(GetConfigdocs, '__init__') as mock_method:
         runner.invoke(shipyard, [auth_vars, 'get', 'configdocs',
                                  '--collection=design'])
-    mock_method.assert_called_once_with(ANY, 'design', 'buffer')
+    mock_method.assert_called_once_with(ANY, 'design', 'buffer', False)
 
 
 def test_get_configdocs_without_passing_collection(*args):