From d62f15123dafa5b77f1bbcedb99b21c2d81023b2 Mon Sep 17 00:00:00 2001 From: Aaron Sheffield Date: Fri, 12 Oct 2018 09:00:32 -0500 Subject: [PATCH] Expects Redacted Raw Documents - Adds a query parameter 'cleartext-secrets' to get full raw documents. - Adds CLI flag to get full raw documents. Change-Id: If38974c8433c8360cc47ae1273720ad76e87a6fd --- doc/source/API.rst | 5 ++++- doc/source/CLI.rst | 5 +++++ .../control/configdocs/configdocs_api.py | 10 +++++++--- .../control/helpers/configdocs_helper.py | 20 ++++++++++++------- .../control/helpers/deckhand_client.py | 12 +++++++++-- .../tests/unit/control/test_configdocs_api.py | 2 +- .../unit/control/test_configdocs_helper.py | 2 +- .../api_client/shipyard_api_client.py | 5 ++++- .../shipyard_client/cli/get/actions.py | 6 ++++-- .../shipyard_client/cli/get/commands.py | 10 ++++++++-- .../tests/unit/cli/get/test_get_commands.py | 2 +- 11 files changed, 58 insertions(+), 21 deletions(-) diff --git a/doc/source/API.rst b/doc/source/API.rst index 130f15b3..79bc8eaa 100644 --- a/doc/source/API.rst +++ b/doc/source/API.rst @@ -220,8 +220,11 @@ Returns the source documents for a collection of documents Query Parameters '''''''''''''''' -version=committed | last_site_action | successful_site_action | **buffer** +- version=committed | last_site_action | successful_site_action | **buffer** Return the documents for the version specified - buffer by default. +- cleartext-secrets=true/**false** + If true then returns cleartext secrets in encrypted documents, otherwise + those values are redacted. Responses ''''''''' diff --git a/doc/source/CLI.rst b/doc/source/CLI.rst index b166503b..2318c792 100644 --- a/doc/source/CLI.rst +++ b/doc/source/CLI.rst @@ -647,6 +647,7 @@ differences between the 'committed' and 'buffer' revision (default behavior). shipyard get configdocs [--collection=] [--committed | --last-site-action | --successful-site-action | --buffer] + [--cleartext-secrets] Example: shipyard get configdocs --collection=design @@ -675,6 +676,10 @@ differences between the 'committed' and 'buffer' revision (default behavior). prior commit. If no documents have been loaded into the buffer for this collection, this will return an empty response (default) +\--cleartext-secrets + Returns cleartext secrets in encrypted documents, otherwise those values + are redacted. Only impacts returned documents, not lists of documents. + Sample ^^^^^^ diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py index 813dcfe3..a778c980 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py @@ -98,11 +98,13 @@ class ConfigDocsResource(BaseResource): Returns a collection of documents """ version = (req.params.get('version') or 'buffer') + cleartext_secrets = req.get_param_as_bool('cleartext-secrets') self._validate_version_parameter(version) helper = ConfigdocsHelper(req.context) # Not reformatting to JSON or YAML since just passing through resp.body = self.get_collection( - helper=helper, collection_id=collection_id, version=version) + helper=helper, collection_id=collection_id, version=version, + cleartext_secrets=cleartext_secrets) resp.append_header('Content-Type', 'application/x-yaml') resp.status = falcon.HTTP_200 @@ -116,13 +118,15 @@ class ConfigDocsResource(BaseResource): status=falcon.HTTP_400, retry=False, ) - def get_collection(self, helper, collection_id, version='buffer'): + def get_collection(self, helper, collection_id, version='buffer', + cleartext_secrets=False): """ Attempts to retrieve the specified collection of documents either from the buffer, committed version, last site action or successful site action, as specified """ - return helper.get_collection_docs(version, collection_id) + return helper.get_collection_docs(version, collection_id, + cleartext_secrets) def post_collection(self, helper, diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py index df6850a3..3ba27223 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py @@ -318,18 +318,21 @@ class ConfigdocsHelper(object): rev = self._get_revision_dict().get(target_revision) return rev['id'] if rev else None - def get_collection_docs(self, version, collection_id): + def get_collection_docs(self, version, collection_id, + cleartext_secrets=False): """ Returns the requested collection of docs based on the version specifier. The default is set as buffer. """ LOG.info('Retrieving collection %s from %s', collection_id, version) if version in [COMMITTED, LAST_SITE_ACTION, SUCCESSFUL_SITE_ACTION]: - return self._get_target_docs(collection_id, version) + return self._get_target_docs(collection_id, version, + cleartext_secrets) - return self._get_doc_from_buffer(collection_id) + return self._get_doc_from_buffer(collection_id, + cleartext_secrets=cleartext_secrets) - def _get_doc_from_buffer(self, collection_id): + def _get_doc_from_buffer(self, collection_id, cleartext_secrets=False): """ Returns the collection if it exists in the buffer. If the buffer contains the collection, the latest @@ -343,7 +346,8 @@ class ConfigdocsHelper(object): # revision exists buffer_id = self.get_revision_id(BUFFER) return self.deckhand.get_docs_from_revision( - revision_id=buffer_id, bucket_id=collection_id) + revision_id=buffer_id, bucket_id=collection_id, + cleartext_secrets=cleartext_secrets) raise ApiError( title='No documents to retrieve', description=('The Shipyard buffer is empty or does not contain ' @@ -351,7 +355,8 @@ class ConfigdocsHelper(object): status=falcon.HTTP_404, retry=False) - def _get_target_docs(self, collection_id, target_rev): + def _get_target_docs(self, collection_id, target_rev, + cleartext_secrets=False): """ Returns the collection if it exists as committed, last_site_action or successful_site_action. @@ -360,7 +365,8 @@ class ConfigdocsHelper(object): if revision_id: return self.deckhand.get_docs_from_revision( - revision_id=revision_id, bucket_id=collection_id) + revision_id=revision_id, bucket_id=collection_id, + cleartext_secrets=cleartext_secrets) raise ApiError( title='No documents to retrieve', diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py index 9c0d4623..4b43028e 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py @@ -184,20 +184,25 @@ class DeckhandClient(object): diff = yaml.safe_load(response.text) return diff - def get_docs_from_revision(self, revision_id, bucket_id=None): + def get_docs_from_revision(self, revision_id, bucket_id=None, + cleartext_secrets=False): """ Retrieves the collection of docs from the revision specified for the bucket_id specified + cleartext_secrets: Should deckhand show or redact secrets. :returns: a string representing the response.text from Deckhand """ + url = DeckhandClient.get_path( DeckhandPaths.REVISION_DOCS ).format(revision_id) # if a bucket_id is specified, limit the response to a bucket - query = None + query = {} if bucket_id is not None: query = {'status.bucket': bucket_id} + if cleartext_secrets is True: + query['cleartext-secrets'] = 'true' response = self._get_request(url, params=query) self._handle_bad_response(response) return response.text @@ -336,6 +341,9 @@ class DeckhandClient(object): 'X-Auth-Token': get_token() } + if not params: + params = None + DeckhandClient._log_request('GET', url, params) response = requests.get( url, diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py index 7704dc5c..969d4170 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py @@ -96,7 +96,7 @@ class TestConfigDocsResource(): helper = ConfigdocsHelper(CTX) cdr.get_collection(helper, 'apples') - mock_method.assert_called_once_with('buffer', 'apples') + mock_method.assert_called_once_with('buffer', 'apples', False) @patch.object(ConfigdocsHelper, 'is_collection_in_buffer', lambda x, y: True) diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py index b34b73f1..3a5cc5a3 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py @@ -503,7 +503,7 @@ def test_get_collection_docs(): """ helper = ConfigdocsHelper(CTX) helper.deckhand.get_docs_from_revision = ( - lambda revision_id, bucket_id: "{'yaml': 'yaml'}") + lambda revision_id, bucket_id, cleartext_secrets: "{'yaml': 'yaml'}") helper._get_revision_dict = lambda: REV_EMPTY_DICT helper.deckhand.get_diff = ( lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT) diff --git a/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py b/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py index d91b06c6..42b578f7 100644 --- a/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py +++ b/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py @@ -68,7 +68,8 @@ class ShipyardClient(BaseClient): ) return self.post_resp(url, query_params, document_data) - def get_configdocs(self, collection_id=None, version='buffer'): + def get_configdocs(self, collection_id=None, version='buffer', + cleartext_secrets=False): """ Get the collection of documents from deckhand specified by collection id @@ -78,6 +79,8 @@ class ShipyardClient(BaseClient): :rtype: Response object """ query_params = {"version": version} + if cleartext_secrets is True: + query_params['cleartext-secrets'] = 'true' url = ApiPaths.POST_GET_CONFIG.value.format( self.get_endpoint(), collection_id) diff --git a/src/bin/shipyard_client/shipyard_client/cli/get/actions.py b/src/bin/shipyard_client/shipyard_client/cli/get/actions.py index 27bc4f85..b8397569 100644 --- a/src/bin/shipyard_client/shipyard_client/cli/get/actions.py +++ b/src/bin/shipyard_client/shipyard_client/cli/get/actions.py @@ -48,7 +48,7 @@ class GetActions(CliAction): class GetConfigdocs(CliAction): """Action to Get Configdocs""" - def __init__(self, ctx, collection, version): + def __init__(self, ctx, collection, version, cleartext_secrets=False): """Sets parameters.""" super().__init__(ctx) self.logger.debug( @@ -56,12 +56,14 @@ class GetConfigdocs(CliAction): "version=%s" % (collection, version)) self.collection = collection self.version = version + self.cleartext_secrets = cleartext_secrets def invoke(self): """Calls API Client and formats response from API Client""" self.logger.debug("Calling API Client get_configdocs.") return self.get_api_client().get_configdocs( - collection_id=self.collection, version=self.version) + collection_id=self.collection, version=self.version, + cleartext_secrets=self.cleartext_secrets) # Handle 404 with default error handler for cli. cli_handled_err_resp_codes = [404] diff --git a/src/bin/shipyard_client/shipyard_client/cli/get/commands.py b/src/bin/shipyard_client/shipyard_client/cli/get/commands.py index fdc6db49..48adc477 100644 --- a/src/bin/shipyard_client/shipyard_client/cli/get/commands.py +++ b/src/bin/shipyard_client/shipyard_client/cli/get/commands.py @@ -98,16 +98,22 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either " flag_value='successful_site_action', help='Holds the revision information for the most recent successfully ' 'executed site action.') +@click.option( + '--cleartext-secrets', + '-t', + help='Returns cleartext secrets in documents', + is_flag=True) @click.pass_context def get_configdocs(ctx, collection, buffer, committed, last_site_action, - successful_site_action): + successful_site_action, cleartext_secrets): if collection: # Get version _version = get_version(ctx, buffer, committed, last_site_action, successful_site_action) click.echo( - GetConfigdocs(ctx, collection, _version).invoke_and_return_resp()) + GetConfigdocs(ctx, collection, _version, + cleartext_secrets).invoke_and_return_resp()) else: compare_versions = check_reformat_versions(ctx, diff --git a/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py b/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py index 17e91ea3..25bf411e 100644 --- a/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py +++ b/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py @@ -58,7 +58,7 @@ def test_get_configdocs_with_passing_collection(*args): with patch.object(GetConfigdocs, '__init__') as mock_method: runner.invoke(shipyard, [auth_vars, 'get', 'configdocs', '--collection=design']) - mock_method.assert_called_once_with(ANY, 'design', 'buffer') + mock_method.assert_called_once_with(ANY, 'design', 'buffer', False) def test_get_configdocs_without_passing_collection(*args):