Additional validation functional tests

Recently the Deckhand Validations API was updated to
return a more nuanced validation error message in
https://review.gerrithub.io/#/c/396251/

This PS adds functional tests to validate the new
format as well as to validate that secrets are
sanitized from the validation error output for
security reasons.

Change-Id: Ic92f2ccd03946a8897aeb2c2a506b3c7eb89836b
This commit is contained in:
Felipe Monteiro 2018-01-31 17:42:46 +00:00
parent 36f752bb93
commit c418e5f5ad
3 changed files with 114 additions and 12 deletions

View File

@ -450,15 +450,15 @@ def _generate_validation_error_output(schema, document, error, root_path):
parent_path_to_error_in_document = '.'.join(
path_to_error_in_document.split('.')[:-1]) or '.'
try:
# NOTE(fmontei): Because validation is performed on fully rendered
# documents, it is necessary to omit the parts of the data section
# where substitution may have occurred to avoid exposing any
# secrets. While this may make debugging a few validation failures
# more difficult, it is a necessary evil.
sanitized_document = SecretsSubstitution.sanitize_potential_secrets(
document)
parent_error_section = utils.jsonpath_parse(
document, parent_path_to_error_in_document)
if 'data' in parent_error_section:
# NOTE(fmontei): Because validation is performed on fully rendered
# documents, it is necessary to omit the parts of the data section
# where substitution may have occurred to avoid exposing any
# secrets. While this may make debugging a few validation failures
# more difficult, it is a necessary evil.
SecretsSubstitution.sanitize_potential_secrets(document)
sanitized_document, parent_path_to_error_in_document)
except Exception:
parent_error_section = (
'Failed to find parent section above where error occurred.')
@ -470,6 +470,8 @@ def _generate_validation_error_output(schema, document, error, root_path):
'schema': document.schema,
'path': path_to_error_in_document,
'error_section': parent_error_section,
# TODO(fmontei): Also sanitize any secrets contained in the message
# as well.
'message': error.message
}

View File

@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
from oslo_log import log as logging
import six
@ -228,7 +230,13 @@ class SecretsSubstitution(object):
:param DocumentDict document: Document to sanitize.
"""
to_sanitize = copy.deepcopy(document)
safe_message = 'Sanitized to avoid exposing secret.'
for sub in document.substitutions:
utils.jsonpath_replace(document['data'], safe_message,
sub['dest']['path'])
replaced_data = utils.jsonpath_replace(
to_sanitize['data'], safe_message, sub['dest']['path'])
if replaced_data:
to_sanitize['data'] = replaced_data
return to_sanitize

View File

@ -7,6 +7,10 @@
# 5. Checks that the document passes schema validation
# 6. Puts an invalid document
# 7. Checks that the document fails schema validation
# 8. Checks that the document entry details adhere to expected validation
# format
# 9. Re-puts the same invalid document with substitutions
# 10. Verify that the substitutions were sanitized in the validation output
defaults:
request_headers:
@ -115,5 +119,93 @@ tests:
response_multidoc_jsonpaths:
$.`len`: 1
$.[0].count: 1
$.[0].results[*].name: deckhand-schema-validation
$.[0].results[*].status: failure
$.[0].results[0].name: deckhand-schema-validation
$.[0].results[0].status: failure
- name: verify_document_validation_failure_entry_details
desc: Check document validation failure details for specific entry
GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation/entries/0
status: 200
response_multidoc_jsonpaths:
$.`len`: 1
$.[0].status: failure
$.[0].errors:
- name: bad
schema: example/Doc/v1
path: .data.b
schema_path: .properties.b.maximum
error_section:
a: this-one-is-required-and-can-be-different
b: 177
message: 177 is greater than the maximum of 100
validation_schema:
$schema: http://json-schema.org/schema#
additionalProperties: False
properties:
a:
type: string
b:
maximum: 100
minimum: 0
type: integer
required:
- a
- b
type: object
- name: add_invalid_document_with_substitutions
desc: Add a document that does not follow the schema
PUT: /api/v1.0/buckets/bad/documents
status: 200
data: |-
schema: example/Doc/v1
metadata:
schema: metadata/Document/v1
name: bad
layeringDefinition:
abstract: false
layer: site
substitutions:
# It doesn't matter if the values below are junk, only that the
# destination path of .a is sanitized in the `error_section` of the
# validation result.
- src:
schema: None
name: None
path: .
dest:
path: .a
data:
a: this-one-is-required-and-can-be-different
b: 177
- name: verify_document_validation_failure_entry_details_hides_secrets
desc: Check document validation failure hides secrets
GET: /api/v1.0/revisions/$HISTORY['add_invalid_document_with_substitutions'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation/entries/0
status: 200
response_multidoc_jsonpaths:
$.`len`: 1
$.[0].status: failure
$.[0].errors:
- name: bad
schema: example/Doc/v1
path: .data.b
schema_path: .properties.b.maximum
error_section:
a: Sanitized to avoid exposing secret.
b: 177
message: 177 is greater than the maximum of 100
validation_schema:
$schema: http://json-schema.org/schema#
additionalProperties: False
properties:
a:
type: string
b:
maximum: 100
minimum: 0
type: integer
required:
- a
- b
type: object