Browse Source

Expects Redacted Raw Documents

- Adds a query parameter 'cleartext-secrets' to get full raw documents.
- Adds CLI flag to get full raw documents.

Change-Id: If38974c8433c8360cc47ae1273720ad76e87a6fd
changes/42/610042/10
Aaron Sheffield 3 years ago
parent
commit
d62f15123d
  1. 5
      doc/source/API.rst
  2. 5
      doc/source/CLI.rst
  3. 10
      src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py
  4. 20
      src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
  5. 12
      src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py
  6. 2
      src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py
  7. 2
      src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py
  8. 5
      src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py
  9. 6
      src/bin/shipyard_client/shipyard_client/cli/get/actions.py
  10. 10
      src/bin/shipyard_client/shipyard_client/cli/get/commands.py
  11. 2
      src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py

5
doc/source/API.rst

@ -220,8 +220,11 @@ Returns the source documents for a collection of documents
Query Parameters
''''''''''''''''
version=committed | last_site_action | successful_site_action | **buffer**
- version=committed | last_site_action | successful_site_action | **buffer**
Return the documents for the version specified - buffer by default.
- cleartext-secrets=true/**false**
If true then returns cleartext secrets in encrypted documents, otherwise
those values are redacted.
Responses
'''''''''

5
doc/source/CLI.rst

@ -647,6 +647,7 @@ differences between the 'committed' and 'buffer' revision (default behavior).
shipyard get configdocs
[--collection=<collection>]
[--committed | --last-site-action | --successful-site-action | --buffer]
[--cleartext-secrets]
Example:
shipyard get configdocs --collection=design
@ -675,6 +676,10 @@ differences between the 'committed' and 'buffer' revision (default behavior).
prior commit. If no documents have been loaded into the buffer for this
collection, this will return an empty response (default)
\--cleartext-secrets
Returns cleartext secrets in encrypted documents, otherwise those values
are redacted. Only impacts returned documents, not lists of documents.
Sample
^^^^^^

10
src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py

@ -98,11 +98,13 @@ class ConfigDocsResource(BaseResource):
Returns a collection of documents
"""
version = (req.params.get('version') or 'buffer')
cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
self._validate_version_parameter(version)
helper = ConfigdocsHelper(req.context)
# Not reformatting to JSON or YAML since just passing through
resp.body = self.get_collection(
helper=helper, collection_id=collection_id, version=version)
helper=helper, collection_id=collection_id, version=version,
cleartext_secrets=cleartext_secrets)
resp.append_header('Content-Type', 'application/x-yaml')
resp.status = falcon.HTTP_200
@ -116,13 +118,15 @@ class ConfigDocsResource(BaseResource):
status=falcon.HTTP_400,
retry=False, )
def get_collection(self, helper, collection_id, version='buffer'):
def get_collection(self, helper, collection_id, version='buffer',
cleartext_secrets=False):
"""
Attempts to retrieve the specified collection of documents
either from the buffer, committed version, last site action
or successful site action, as specified
"""
return helper.get_collection_docs(version, collection_id)
return helper.get_collection_docs(version, collection_id,
cleartext_secrets)
def post_collection(self,
helper,

20
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py

@ -318,18 +318,21 @@ class ConfigdocsHelper(object):
rev = self._get_revision_dict().get(target_revision)
return rev['id'] if rev else None
def get_collection_docs(self, version, collection_id):
def get_collection_docs(self, version, collection_id,
cleartext_secrets=False):
"""
Returns the requested collection of docs based on the version
specifier. The default is set as buffer.
"""
LOG.info('Retrieving collection %s from %s', collection_id, version)
if version in [COMMITTED, LAST_SITE_ACTION, SUCCESSFUL_SITE_ACTION]:
return self._get_target_docs(collection_id, version)
return self._get_target_docs(collection_id, version,
cleartext_secrets)
return self._get_doc_from_buffer(collection_id)
return self._get_doc_from_buffer(collection_id,
cleartext_secrets=cleartext_secrets)
def _get_doc_from_buffer(self, collection_id):
def _get_doc_from_buffer(self, collection_id, cleartext_secrets=False):
"""
Returns the collection if it exists in the buffer.
If the buffer contains the collection, the latest
@ -343,7 +346,8 @@ class ConfigdocsHelper(object):
# revision exists
buffer_id = self.get_revision_id(BUFFER)
return self.deckhand.get_docs_from_revision(
revision_id=buffer_id, bucket_id=collection_id)
revision_id=buffer_id, bucket_id=collection_id,
cleartext_secrets=cleartext_secrets)
raise ApiError(
title='No documents to retrieve',
description=('The Shipyard buffer is empty or does not contain '
@ -351,7 +355,8 @@ class ConfigdocsHelper(object):
status=falcon.HTTP_404,
retry=False)
def _get_target_docs(self, collection_id, target_rev):
def _get_target_docs(self, collection_id, target_rev,
cleartext_secrets=False):
"""
Returns the collection if it exists as committed, last_site_action
or successful_site_action.
@ -360,7 +365,8 @@ class ConfigdocsHelper(object):
if revision_id:
return self.deckhand.get_docs_from_revision(
revision_id=revision_id, bucket_id=collection_id)
revision_id=revision_id, bucket_id=collection_id,
cleartext_secrets=cleartext_secrets)
raise ApiError(
title='No documents to retrieve',

12
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py

@ -184,20 +184,25 @@ class DeckhandClient(object):
diff = yaml.safe_load(response.text)
return diff
def get_docs_from_revision(self, revision_id, bucket_id=None):
def get_docs_from_revision(self, revision_id, bucket_id=None,
cleartext_secrets=False):
"""
Retrieves the collection of docs from the revision specified
for the bucket_id specified
cleartext_secrets: Should deckhand show or redact secrets.
:returns: a string representing the response.text from Deckhand
"""
url = DeckhandClient.get_path(
DeckhandPaths.REVISION_DOCS
).format(revision_id)
# if a bucket_id is specified, limit the response to a bucket
query = None
query = {}
if bucket_id is not None:
query = {'status.bucket': bucket_id}
if cleartext_secrets is True:
query['cleartext-secrets'] = 'true'
response = self._get_request(url, params=query)
self._handle_bad_response(response)
return response.text
@ -336,6 +341,9 @@ class DeckhandClient(object):
'X-Auth-Token': get_token()
}
if not params:
params = None
DeckhandClient._log_request('GET', url, params)
response = requests.get(
url,

2
src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py

@ -96,7 +96,7 @@ class TestConfigDocsResource():
helper = ConfigdocsHelper(CTX)
cdr.get_collection(helper, 'apples')
mock_method.assert_called_once_with('buffer', 'apples')
mock_method.assert_called_once_with('buffer', 'apples', False)
@patch.object(ConfigdocsHelper, 'is_collection_in_buffer',
lambda x, y: True)

2
src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py

@ -503,7 +503,7 @@ def test_get_collection_docs():
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_docs_from_revision = (
lambda revision_id, bucket_id: "{'yaml': 'yaml'}")
lambda revision_id, bucket_id, cleartext_secrets: "{'yaml': 'yaml'}")
helper._get_revision_dict = lambda: REV_EMPTY_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT)

5
src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py

@ -68,7 +68,8 @@ class ShipyardClient(BaseClient):
)
return self.post_resp(url, query_params, document_data)
def get_configdocs(self, collection_id=None, version='buffer'):
def get_configdocs(self, collection_id=None, version='buffer',
cleartext_secrets=False):
"""
Get the collection of documents from deckhand specified by
collection id
@ -78,6 +79,8 @@ class ShipyardClient(BaseClient):
:rtype: Response object
"""
query_params = {"version": version}
if cleartext_secrets is True:
query_params['cleartext-secrets'] = 'true'
url = ApiPaths.POST_GET_CONFIG.value.format(
self.get_endpoint(),
collection_id)

6
src/bin/shipyard_client/shipyard_client/cli/get/actions.py

@ -48,7 +48,7 @@ class GetActions(CliAction):
class GetConfigdocs(CliAction):
"""Action to Get Configdocs"""
def __init__(self, ctx, collection, version):
def __init__(self, ctx, collection, version, cleartext_secrets=False):
"""Sets parameters."""
super().__init__(ctx)
self.logger.debug(
@ -56,12 +56,14 @@ class GetConfigdocs(CliAction):
"version=%s" % (collection, version))
self.collection = collection
self.version = version
self.cleartext_secrets = cleartext_secrets
def invoke(self):
"""Calls API Client and formats response from API Client"""
self.logger.debug("Calling API Client get_configdocs.")
return self.get_api_client().get_configdocs(
collection_id=self.collection, version=self.version)
collection_id=self.collection, version=self.version,
cleartext_secrets=self.cleartext_secrets)
# Handle 404 with default error handler for cli.
cli_handled_err_resp_codes = [404]

10
src/bin/shipyard_client/shipyard_client/cli/get/commands.py

@ -98,16 +98,22 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either "
flag_value='successful_site_action',
help='Holds the revision information for the most recent successfully '
'executed site action.')
@click.option(
'--cleartext-secrets',
'-t',
help='Returns cleartext secrets in documents',
is_flag=True)
@click.pass_context
def get_configdocs(ctx, collection, buffer, committed, last_site_action,
successful_site_action):
successful_site_action, cleartext_secrets):
if collection:
# Get version
_version = get_version(ctx, buffer, committed, last_site_action,
successful_site_action)
click.echo(
GetConfigdocs(ctx, collection, _version).invoke_and_return_resp())
GetConfigdocs(ctx, collection, _version,
cleartext_secrets).invoke_and_return_resp())
else:
compare_versions = check_reformat_versions(ctx,

2
src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py

@ -58,7 +58,7 @@ def test_get_configdocs_with_passing_collection(*args):
with patch.object(GetConfigdocs, '__init__') as mock_method:
runner.invoke(shipyard, [auth_vars, 'get', 'configdocs',
'--collection=design'])
mock_method.assert_called_once_with(ANY, 'design', 'buffer')
mock_method.assert_called_once_with(ANY, 'design', 'buffer', False)
def test_get_configdocs_without_passing_collection(*args):

Loading…
Cancel
Save