Browse Source
Refactors the uwsgi-based logging filter to encapsulate better and adds a redaction filter for api logging. This change now also includes bringing a stray test back into the correct location. Change-Id: I53de7da6bd4182e824366fdbb5221c7c9ae5be09changes/41/569841/2
Bryan Strassner
4 years ago
17 changed files with 544 additions and 200 deletions
@ -0,0 +1,84 @@ |
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
"""Container for logging configuration""" |
|||
import logging |
|||
|
|||
from shipyard_airflow.control.logging import request_logging |
|||
from shipyard_airflow.control.logging.redaction_formatter import ( |
|||
RedactionFormatter |
|||
) |
|||
LOG = logging.getLogger(__name__) |
|||
|
|||
|
|||
class LoggingConfig(): |
|||
"""Encapsulates the properties of a Logging Config |
|||
|
|||
:param level: The level value to set as the threshold for logging. Ideally |
|||
a client would use logging.INFO or the desired logging constant to set |
|||
this level value |
|||
:param named_levels: A dictionary of 'name': logging.level values that |
|||
configures the minimum logging level for the named logger. |
|||
:param format_string: Optional value allowing for override of the logging |
|||
format string. If new values beyond the default value are introduced, |
|||
the additional_fields must contain those fields to ensure they are set |
|||
upon using the logging filter. |
|||
:param additional_fields: Optionally allows for specifying more fields that |
|||
will be set on each logging record. If specified, the format_string |
|||
parameter should be set with matching fields, otherwise they will not |
|||
be displayed. |
|||
""" |
|||
|
|||
_default_log_format = ( |
|||
"%(asctime)s %(levelname)-8s %(req_id)s %(external_ctx)s %(user)s " |
|||
"%(module)s(%(lineno)d) %(funcName)s - %(message)s") |
|||
|
|||
def __init__(self, |
|||
level, |
|||
named_levels=None, |
|||
format_string=None, |
|||
additional_fields=None): |
|||
self.level = level |
|||
# Any false values passed for named_levels should instead be treated as |
|||
# an empty dictionary i.e. no special log levels |
|||
self.named_levels = named_levels or {} |
|||
# Any false values for format string should use the default instead |
|||
self.format_string = format_string or LoggingConfig._default_log_format |
|||
self.additional_fields = additional_fields |
|||
|
|||
def setup_logging(self): |
|||
""" Establishes the base logging using the appropriate filter |
|||
attached to the console/stream handler. |
|||
""" |
|||
console_handler = logging.StreamHandler() |
|||
request_logging.assign_request_filter(console_handler, |
|||
self.additional_fields) |
|||
logging.basicConfig(level=self.level, |
|||
format=self.format_string, |
|||
handlers=[console_handler]) |
|||
for handler in logging.root.handlers: |
|||
handler.setFormatter(RedactionFormatter(handler.formatter)) |
|||
logger = logging.getLogger(__name__) |
|||
logger.info('Established logging defaults') |
|||
self._setup_log_levels() |
|||
|
|||
def _setup_log_levels(self): |
|||
"""Sets up the logger levels for named loggers |
|||
|
|||
:param named_levels: dict to set each of the specified |
|||
logging levels. |
|||
""" |
|||
for logger_name, level in self.named_levels.items(): |
|||
logger = logging.getLogger(logger_name) |
|||
logger.setLevel(level) |
|||
LOG.info("Set %s to use logging level %s", logger_name, level) |
|||
@ -0,0 +1,59 @@ |
|||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
"""A logging formatter that attempts to scrub logged data. |
|||
|
|||
Addresses the OWASP considerations of things that should be excluded: |
|||
https://www.owasp.org/index.php/Logging_Cheat_Sheet#Data_to_exclude |
|||
|
|||
These that should not usually be logged: |
|||
|
|||
Avoided by circumstance - no action taken by this formatter: |
|||
- Application source code |
|||
- Avoided on a case by case basis |
|||
- Session identification values (consider replacing with a hashed value if |
|||
needed to track session specific events) |
|||
- Stateless application logs a transaction correlation ID, but not a |
|||
replayable session id. |
|||
- Access tokens |
|||
- Headers with access tokens are excluded in the request/response logging |
|||
|
|||
Formatter provides scrubbing facilities: |
|||
- Authentication passwords |
|||
- Database connection strings |
|||
- Encryption keys and other master secrets |
|||
""" |
|||
import logging |
|||
|
|||
from shipyard_airflow.control.util.redactor import Redactor |
|||
|
|||
|
|||
LOG = logging.getLogger(__name__) |
|||
# Concepts from https://www.relaxdiego.com/2014/07/logging-in-python.html |
|||
|
|||
|
|||
class RedactionFormatter(object): |
|||
""" A formatter to remove sensitive information from logs |
|||
""" |
|||
def __init__(self, original_formatter): |
|||
self.original_formatter = original_formatter |
|||
self.redactor = Redactor() |
|||
LOG.info("RedactionFormatter wrapping %s", |
|||
original_formatter.__class__.__name__) |
|||
|
|||
def format(self, record): |
|||
msg = self.original_formatter.format(record) |
|||
return self.redactor.redact(msg) |
|||
|
|||
def __getattr__(self, attr): |
|||
return getattr(self.original_formatter, attr) |
|||
@ -0,0 +1,76 @@ |
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
""" A logging filter to prepend request scoped formatting to all logging |
|||
records that use this filter. Request-based values will cause the log |
|||
records to have correlation values that can be used to better trace |
|||
logs. If no handler that supports a request scope is present, does not attempt |
|||
to change the logs in any way |
|||
|
|||
Threads initiated using threading. Thread can be correlated to the request |
|||
they came from by setting a kwarg of log_extra, containing a dictionary |
|||
of values matching the VALID_ADDL_FIELDS below and any fields that are set |
|||
as additional_fields by the setup_logging function. This mechanism assumes |
|||
that the thread will maintain the correlation values for the life |
|||
of the thread. |
|||
""" |
|||
import logging |
|||
|
|||
# Import uwsgi to determine if it has been provided to the application. |
|||
# Only import the UwsgiLogFilter if uwsgi is present |
|||
try: |
|||
import uwsgi |
|||
from shipyard_airflow.control.logging.uwsgi_filter import UwsgiLogFilter |
|||
except ImportError: |
|||
uwsgi = None |
|||
|
|||
|
|||
# BASE_ADDL_FIELDS are fields that will be included in the request based |
|||
# logging - these fields need not be set up independently as opposed to the |
|||
# additional_fields parameter used below, which allows for more fields beyond |
|||
# this default set. |
|||
BASE_ADDL_FIELDS = ['req_id', 'external_ctx', 'user'] |
|||
LOG = logging.getLogger(__name__) |
|||
|
|||
|
|||
def set_logvar(key, value): |
|||
""" Attempts to set the logvar in the request scope, or ignores it |
|||
if not running in an environment that supports it. |
|||
""" |
|||
if value: |
|||
if uwsgi: |
|||
uwsgi.set_logvar(key, value) |
|||
# If there were a different handler than uwsgi, here is where we'd |
|||
# need to set the logvar value for its use. |
|||
|
|||
|
|||
def assign_request_filter(handler, additional_fields=None): |
|||
"""Adds the request-scoped filter log filter to the passed handler |
|||
|
|||
:param handler: a logging handler, e.g. a ConsoleHandler |
|||
:param additional_fields: fields that will be included in the logging |
|||
records (if a matching logging format is used) |
|||
""" |
|||
handler_cls = handler.__class__.__name__ |
|||
if uwsgi: |
|||
if additional_fields is None: |
|||
additional_fields = [] |
|||
addl_fields = [*BASE_ADDL_FIELDS, *additional_fields] |
|||
handler.addFilter(UwsgiLogFilter(uwsgi, addl_fields)) |
|||
LOG.info("UWSGI present, added UWSGI log filter to handler %s", |
|||
handler_cls) |
|||
# if there are other handlers that would allow for request scoped logging |
|||
# to be set up, we could include those options here. |
|||
else: |
|||
LOG.info("No request based logging filter in the current environment. " |
|||
"No log filter added to handler %s", handler_cls) |
|||
@ -0,0 +1,76 @@ |
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
"""A UWSGI-dependent implementation of a logging filter allowing for |
|||
request-based logging. |
|||
""" |
|||
import logging |
|||
import threading |
|||
|
|||
|
|||
class UwsgiLogFilter(logging.Filter): |
|||
""" A filter that preepends log records with additional request |
|||
based information, or information provided by log_extra in the |
|||
kwargs provided to a thread |
|||
""" |
|||
def __init__(self, uwsgi, additional_fields=None): |
|||
super().__init__() |
|||
if additional_fields is None: |
|||
additional_fields = [] |
|||
self.uwsgi = uwsgi |
|||
self.log_fields = additional_fields |
|||
|
|||
def filter(self, record): |
|||
""" Checks for thread provided values, or attempts to get values |
|||
from uwsgi |
|||
""" |
|||
if self._thread_has_log_extra(): |
|||
value_setter = self._set_values_from_log_extra |
|||
else: |
|||
value_setter = self._set_value |
|||
|
|||
for field_nm in self.log_fields: |
|||
value_setter(record, field_nm) |
|||
return True |
|||
|
|||
def _set_value(self, record, logvar): |
|||
# handles setting the logvars from uwsgi or '' in case of none/empty |
|||
try: |
|||
logvar_value = self.uwsgi.get_logvar(logvar) |
|||
if logvar_value: |
|||
setattr(record, logvar, logvar_value.decode('UTF-8')) |
|||
else: |
|||
setattr(record, logvar, '') |
|||
except SystemError: |
|||
# This happens if log_extra is not on a thread that is spawned |
|||
# by a process running under uwsgi |
|||
setattr(record, logvar, '') |
|||
|
|||
def _set_values_from_log_extra(self, record, logvar): |
|||
# sets the values from the log_extra on the thread |
|||
setattr(record, logvar, self._get_value_from_thread(logvar) or '') |
|||
|
|||
def _thread_has_log_extra(self): |
|||
# Checks to see if log_extra is present on the current thread |
|||
if self._get_log_extra_from_thread(): |
|||
return True |
|||
return False |
|||
|
|||
def _get_value_from_thread(self, logvar): |
|||
# retrieve the logvar from the log_extra from kwargs for the thread |
|||
return self._get_log_extra_from_thread().get(logvar, '') |
|||
|
|||
def _get_log_extra_from_thread(self): |
|||
# retrieves the log_extra value from kwargs or {} if it doesn't |
|||
# exist |
|||
return threading.current_thread()._kwargs.get('log_extra', {}) |
|||
@ -1,147 +0,0 @@ |
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
""" A logging filter to prepend UWSGI-handled formatting to all logging |
|||
records that use this filter. Request-based values will cause the log |
|||
records to have correlation values that can be used to better trace |
|||
logs. If uwsgi is not present, does not attempt to change the logs in |
|||
any way |
|||
|
|||
Threads initiated using threading.Thread can be correlated to the request |
|||
they came from by setting a kwarg of log_extra, containing a dictionary |
|||
of valeus matching the VALID_ADDL_FIELDS below and any fields that are set |
|||
as additional_fields by the setup_logging function. This mechanism assumes |
|||
that the thread will maintain the correlation values for the life |
|||
of the thread. |
|||
""" |
|||
import logging |
|||
import threading |
|||
|
|||
# Import uwsgi to determine if it has been provided to the application. |
|||
try: |
|||
import uwsgi |
|||
except ImportError: |
|||
uwsgi = None |
|||
|
|||
VALID_ADDL_FIELDS = ['req_id', 'external_ctx', 'user'] |
|||
_DEFAULT_LOG_FORMAT = ( |
|||
'%(asctime)s %(levelname)-8s %(req_id)s %(external_ctx)s %(user)s ' |
|||
'%(module)s(%(lineno)d) %(funcName)s - %(message)s' |
|||
) |
|||
|
|||
_LOG_FORMAT_IN_USE = None |
|||
|
|||
|
|||
def setup_logging(level, format_string=None, additional_fields=None): |
|||
""" Establishes the base logging using the appropriate filter |
|||
attached to the console/stream handler. |
|||
:param level: The level value to set as the threshold for |
|||
logging. Ideally a client would use logging.INFO or |
|||
the desired logging constant to set this level value |
|||
:param format_string: Optional value allowing for override of the |
|||
logging format string. If new values beyond |
|||
the default value are introduced, the |
|||
additional_fields must contain those fields |
|||
to ensure they are set upon using the logging |
|||
filter. |
|||
:param additional_fields: Optionally allows for specifying more |
|||
fields that will be set on each logging |
|||
record. If specified, the format_string |
|||
parameter should be set with matching |
|||
fields, otherwise they will not be |
|||
displayed. |
|||
""" |
|||
global _LOG_FORMAT_IN_USE |
|||
_LOG_FORMAT_IN_USE = format_string or _DEFAULT_LOG_FORMAT |
|||
|
|||
console_handler = logging.StreamHandler() |
|||
if uwsgi: |
|||
console_handler.addFilter(UwsgiLogFilter(additional_fields)) |
|||
logging.basicConfig(level=level, |
|||
format=_LOG_FORMAT_IN_USE, |
|||
handlers=[console_handler]) |
|||
logger = logging.getLogger(__name__) |
|||
logger.info('Established logging defaults') |
|||
|
|||
|
|||
def get_log_format(): |
|||
""" Returns the common log format being used by this application |
|||
""" |
|||
return _LOG_FORMAT_IN_USE |
|||
|
|||
|
|||
def set_logvar(key, value): |
|||
""" Attempts to set the logvar in the request scope , or ignores it |
|||
if not running in uwsgi |
|||
""" |
|||
if uwsgi and value: |
|||
uwsgi.set_logvar(key, value) |
|||
|
|||
|
|||
class UwsgiLogFilter(logging.Filter): |
|||
""" A filter that preepends log records with additional request |
|||
based information, or information provided by log_extra in the |
|||
kwargs provided to a thread |
|||
""" |
|||
def __init__(self, additional_fields=None): |
|||
super().__init__() |
|||
if additional_fields is None: |
|||
additional_fields = [] |
|||
self.log_fields = [*VALID_ADDL_FIELDS, *additional_fields] |
|||
|
|||
def filter(self, record): |
|||
""" Checks for thread provided values, or attempts to get values |
|||
from uwsgi |
|||
""" |
|||
if self._thread_has_log_extra(): |
|||
value_setter = self._set_values_from_log_extra |
|||
else: |
|||
value_setter = self._set_value |
|||
|
|||
for field_nm in self.log_fields: |
|||
value_setter(record, field_nm) |
|||
return True |
|||
|
|||
def _set_value(self, record, logvar): |
|||
# handles setting the logvars from uwsgi or '' in case of none/empty |
|||
try: |
|||
logvar_value = None |
|||
if uwsgi: |
|||
logvar_value = uwsgi.get_logvar(logvar) |
|||
if logvar_value: |
|||
setattr(record, logvar, logvar_value.decode('UTF-8')) |
|||
else: |
|||
setattr(record, logvar, '') |
|||
except SystemError: |
|||
# This happens if log_extra is not on a thread that is spawned |
|||
# by a process running under uwsgi |
|||
setattr(record, logvar, '') |
|||
|
|||
def _set_values_from_log_extra(self, record, logvar): |
|||
# sets the values from the log_extra on the thread |
|||
setattr(record, logvar, self._get_value_from_thread(logvar) or '') |
|||
|
|||
def _thread_has_log_extra(self): |
|||
# Checks to see if log_extra is present on the current thread |
|||
if self._get_log_extra_from_thread(): |
|||
return True |
|||
return False |
|||
|
|||
def _get_value_from_thread(self, logvar): |
|||
# retrieve the logvar from the log_extra from kwargs for the thread |
|||
return self._get_log_extra_from_thread().get(logvar, '') |
|||
|
|||
def _get_log_extra_from_thread(self): |
|||
# retrieves the log_extra value from kwargs or {} if it doesn't |
|||
# exist |
|||
return threading.current_thread()._kwargs.get('log_extra', {}) |
|||
@ -0,0 +1,119 @@ |
|||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
"""String redaction based upon regex patterns |
|||
|
|||
Uses the keys and patterns from oslo_utils as a starting point, and layers in |
|||
new keys and patterns defined locally. |
|||
""" |
|||
import re |
|||
|
|||
# oslo_utils makes the choice to keep the list of supported redactions |
|||
# non-extensible (purposefully). Here we can define new values, and follow the |
|||
# same basic logic as used in strutils. |
|||
# The following are copied from oslo_utils, strutils. Extend using the other |
|||
# facilties to make these easier to keep in sync: |
|||
_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+'] |
|||
_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\'])[^\"\']*([\"\'])', |
|||
r'(%(key)s\s+[\"\'])[^\"\']*([\"\'])', |
|||
r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)', |
|||
r'(<%(key)s>)[^<]*(</%(key)s>)', |
|||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\'])[^\"\']*([\"\'])', |
|||
r'([\'"][^"\']*%(key)s[\'"]\s*:\s*u?[\'"])[^\"\']*' |
|||
'([\'"])', |
|||
r'([\'"][^\'"]*%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?' |
|||
'[\'"])[^\"\']*([\'"])', |
|||
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] |
|||
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password', |
|||
'auth_token', 'new_pass', 'auth_password', 'secret_uuid', |
|||
'secret', 'sys_pswd', 'token', 'configdrive', |
|||
'CHAPPASSWORD', 'encrypted_key'] |
|||
|
|||
|
|||
class Redactor(): |
|||
"""String redactor that can be used to replace sensitive values |
|||
|
|||
:param redaction: the string value to put in place in case of a redaction |
|||
:param keys: list of additional keys that are searched for and have related |
|||
values redacted |
|||
:param single_patterns: list of additional single capture group regex |
|||
patterns |
|||
:param double_patterns: list of additional double capture group regex |
|||
patterns |
|||
""" |
|||
# Start with the values defined in strutils |
|||
_KEYS = list(_SANITIZE_KEYS) |
|||
|
|||
_SINGLE_CG_PATTERNS = list(_FORMAT_PATTERNS_1) |
|||
_DOUBLE_CG_PATTERNS = list(_FORMAT_PATTERNS_2) |
|||
|
|||
# More keys to extend the set of keys used in identifying redactions |
|||
_KEYS.extend([]) |
|||
# More single capture group patterns |
|||
_SINGLE_CG_PATTERNS.extend([r'(%(key)s\s*[:]\s*)[^\s^\'^\"]+']) |
|||
# More two capture group patterns |
|||
_DOUBLE_CG_PATTERNS.extend([]) |
|||
|
|||
def __init__(self, |
|||
redaction='***', |
|||
keys=None, |
|||
single_patterns=None, |
|||
double_patterns=None): |
|||
if keys is None: |
|||
keys = [] |
|||
if single_patterns is None: |
|||
single_patterns = [] |
|||
if double_patterns is None: |
|||
double_patterns = [] |
|||
|
|||
self.redaction = redaction |
|||
|
|||
self.keys = list(Redactor._KEYS) |
|||
self.keys.extend(keys) |
|||
|
|||
singles = list(Redactor._SINGLE_CG_PATTERNS) |
|||
singles.extend(single_patterns) |
|||
|
|||
doubles = list(Redactor._DOUBLE_CG_PATTERNS) |
|||
doubles.extend(double_patterns) |
|||
|
|||
self._single_cg_patterns = self._gen_patterns(patterns=singles) |
|||
self._double_cg_patterns = self._gen_patterns(patterns=doubles) |
|||
# the two capture group patterns |
|||
|
|||
def _gen_patterns(self, patterns): |
|||
"""Initialize the redaction patterns""" |
|||
regex_patterns = {} |
|||
for key in self.keys: |
|||
regex_patterns[key] = [] |
|||
for pattern in patterns: |
|||
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) |
|||
regex_patterns[key].append(reg_ex) |
|||
return regex_patterns |
|||
|
|||
def redact(self, message): |
|||
"""Apply regex based replacements to mask values |
|||
|
|||
Modeled from: |
|||
https://github.com/openstack/oslo.utils/blob/9b23c17a6be6d07d171b64ada3629c3680598f7b/oslo_utils/strutils.py#L272 |
|||
""" |
|||
substitute1 = r'\g<1>' + self.redaction |
|||
substitute2 = r'\g<1>' + self.redaction + r'\g<2>' |
|||
|
|||
for key in self.keys: |
|||
if key in message: |
|||
for pattern in self._double_cg_patterns[key]: |
|||
message = re.sub(pattern, substitute2, message) |
|||
for pattern in self._single_cg_patterns[key]: |
|||
message = re.sub(pattern, substitute1, message) |
|||
return message |
|||
@ -0,0 +1,106 @@ |
|||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
"""Tests for redaction functionality |
|||
|
|||
- redaction_formatter |
|||
- redactor |
|||
""" |
|||
import logging |
|||
import sys |
|||
|
|||
from shipyard_airflow.control.logging.redaction_formatter import ( |
|||
RedactionFormatter |
|||
) |
|||
from shipyard_airflow.control.util.redactor import Redactor |
|||
|
|||
|
|||
class TestRedaction(): |
|||
|
|||
def test_redactor(self): |
|||
redactor = Redactor() |
|||
|
|||
to_redact = "My password = swordfish" |
|||
expected = "My password = ***" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
# not a detected pattern - should be same |
|||
to_redact = "My pass = swordfish" |
|||
expected = "My pass = swordfish" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
# yaml format |
|||
to_redact = "My password: swordfish" |
|||
expected = "My password: ***" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
# yaml format |
|||
to_redact = "My password: swordfish is not for sale" |
|||
expected = "My password: *** is not for sale" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
to_redact = """ |
|||
password: |
|||
swordfish |
|||
""" |
|||
expected = """ |
|||
password: |
|||
*** |
|||
""" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
to_redact = """ |
|||
password: |
|||
swordfish |
|||
trains: |
|||
cheese |
|||
""" |
|||
expected = """ |
|||
password: |
|||
*** |
|||
trains: |
|||
cheese |
|||
""" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
def test_extended_keys_redactor(self): |
|||
redactor = Redactor(redaction="++++", keys=['trains']) |
|||
to_redact = """ |
|||
password: |
|||
swordfish |
|||
trains: |
|||
cheese |
|||
""" |
|||
expected = """ |
|||
password: |
|||
++++ |
|||
trains: |
|||
++++ |
|||
""" |
|||
assert redactor.redact(to_redact) == expected |
|||
|
|||
def test_redaction_formatter(self, caplog): |
|||
# since root logging is setup by prior tests need to remove all |
|||
# handlers to simulate a clean environment of setting up this |
|||
# RedactionFormatter |
|||
for handler in list(logging.root.handlers): |
|||
if not "LogCaptureHandler" == handler.__class__.__name__: |
|||
logging.root.removeHandler(handler) |
|||
|
|||
logging.basicConfig(level=logging.DEBUG, |
|||
handlers=[logging.StreamHandler(sys.stdout)]) |
|||
for handler in logging.root.handlers: |
|||
handler.setFormatter(RedactionFormatter(handler.formatter)) |
|||
logging.info('Established password: albatross for this test') |
|||
assert 'albatross' not in caplog.text |
|||
assert 'Established password: *** for this test' in caplog.text |
|||
Loading…
Reference in new issue