Browse Source
Refactors the uwsgi-based logging filter to encapsulate better and adds a redaction filter for api logging. This change now also includes bringing a stray test back into the correct location. Change-Id: I53de7da6bd4182e824366fdbb5221c7c9ae5be09changes/41/569841/2
Bryan Strassner
4 years ago
17 changed files with 544 additions and 200 deletions
@ -0,0 +1,84 @@
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Container for logging configuration""" |
||||
import logging |
||||
|
||||
from shipyard_airflow.control.logging import request_logging |
||||
from shipyard_airflow.control.logging.redaction_formatter import ( |
||||
RedactionFormatter |
||||
) |
||||
LOG = logging.getLogger(__name__) |
||||
|
||||
|
||||
class LoggingConfig(): |
||||
"""Encapsulates the properties of a Logging Config |
||||
|
||||
:param level: The level value to set as the threshold for logging. Ideally |
||||
a client would use logging.INFO or the desired logging constant to set |
||||
this level value |
||||
:param named_levels: A dictionary of 'name': logging.level values that |
||||
configures the minimum logging level for the named logger. |
||||
:param format_string: Optional value allowing for override of the logging |
||||
format string. If new values beyond the default value are introduced, |
||||
the additional_fields must contain those fields to ensure they are set |
||||
upon using the logging filter. |
||||
:param additional_fields: Optionally allows for specifying more fields that |
||||
will be set on each logging record. If specified, the format_string |
||||
parameter should be set with matching fields, otherwise they will not |
||||
be displayed. |
||||
""" |
||||
|
||||
_default_log_format = ( |
||||
"%(asctime)s %(levelname)-8s %(req_id)s %(external_ctx)s %(user)s " |
||||
"%(module)s(%(lineno)d) %(funcName)s - %(message)s") |
||||
|
||||
def __init__(self, |
||||
level, |
||||
named_levels=None, |
||||
format_string=None, |
||||
additional_fields=None): |
||||
self.level = level |
||||
# Any false values passed for named_levels should instead be treated as |
||||
# an empty dictionary i.e. no special log levels |
||||
self.named_levels = named_levels or {} |
||||
# Any false values for format string should use the default instead |
||||
self.format_string = format_string or LoggingConfig._default_log_format |
||||
self.additional_fields = additional_fields |
||||
|
||||
def setup_logging(self): |
||||
""" Establishes the base logging using the appropriate filter |
||||
attached to the console/stream handler. |
||||
""" |
||||
console_handler = logging.StreamHandler() |
||||
request_logging.assign_request_filter(console_handler, |
||||
self.additional_fields) |
||||
logging.basicConfig(level=self.level, |
||||
format=self.format_string, |
||||
handlers=[console_handler]) |
||||
for handler in logging.root.handlers: |
||||
handler.setFormatter(RedactionFormatter(handler.formatter)) |
||||
logger = logging.getLogger(__name__) |
||||
logger.info('Established logging defaults') |
||||
self._setup_log_levels() |
||||
|
||||
def _setup_log_levels(self): |
||||
"""Sets up the logger levels for named loggers |
||||
|
||||
:param named_levels: dict to set each of the specified |
||||
logging levels. |
||||
""" |
||||
for logger_name, level in self.named_levels.items(): |
||||
logger = logging.getLogger(logger_name) |
||||
logger.setLevel(level) |
||||
LOG.info("Set %s to use logging level %s", logger_name, level) |
||||
@ -0,0 +1,59 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""A logging formatter that attempts to scrub logged data. |
||||
|
||||
Addresses the OWASP considerations of things that should be excluded: |
||||
https://www.owasp.org/index.php/Logging_Cheat_Sheet#Data_to_exclude |
||||
|
||||
These that should not usually be logged: |
||||
|
||||
Avoided by circumstance - no action taken by this formatter: |
||||
- Application source code |
||||
- Avoided on a case by case basis |
||||
- Session identification values (consider replacing with a hashed value if |
||||
needed to track session specific events) |
||||
- Stateless application logs a transaction correlation ID, but not a |
||||
replayable session id. |
||||
- Access tokens |
||||
- Headers with access tokens are excluded in the request/response logging |
||||
|
||||
Formatter provides scrubbing facilities: |
||||
- Authentication passwords |
||||
- Database connection strings |
||||
- Encryption keys and other master secrets |
||||
""" |
||||
import logging |
||||
|
||||
from shipyard_airflow.control.util.redactor import Redactor |
||||
|
||||
|
||||
LOG = logging.getLogger(__name__) |
||||
# Concepts from https://www.relaxdiego.com/2014/07/logging-in-python.html |
||||
|
||||
|
||||
class RedactionFormatter(object): |
||||
""" A formatter to remove sensitive information from logs |
||||
""" |
||||
def __init__(self, original_formatter): |
||||
self.original_formatter = original_formatter |
||||
self.redactor = Redactor() |
||||
LOG.info("RedactionFormatter wrapping %s", |
||||
original_formatter.__class__.__name__) |
||||
|
||||
def format(self, record): |
||||
msg = self.original_formatter.format(record) |
||||
return self.redactor.redact(msg) |
||||
|
||||
def __getattr__(self, attr): |
||||
return getattr(self.original_formatter, attr) |
||||
@ -0,0 +1,76 @@
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
""" A logging filter to prepend request scoped formatting to all logging |
||||
records that use this filter. Request-based values will cause the log |
||||
records to have correlation values that can be used to better trace |
||||
logs. If no handler that supports a request scope is present, does not attempt |
||||
to change the logs in any way |
||||
|
||||
Threads initiated using threading. Thread can be correlated to the request |
||||
they came from by setting a kwarg of log_extra, containing a dictionary |
||||
of values matching the VALID_ADDL_FIELDS below and any fields that are set |
||||
as additional_fields by the setup_logging function. This mechanism assumes |
||||
that the thread will maintain the correlation values for the life |
||||
of the thread. |
||||
""" |
||||
import logging |
||||
|
||||
# Import uwsgi to determine if it has been provided to the application. |
||||
# Only import the UwsgiLogFilter if uwsgi is present |
||||
try: |
||||
import uwsgi |
||||
from shipyard_airflow.control.logging.uwsgi_filter import UwsgiLogFilter |
||||
except ImportError: |
||||
uwsgi = None |
||||
|
||||
|
||||
# BASE_ADDL_FIELDS are fields that will be included in the request based |
||||
# logging - these fields need not be set up independently as opposed to the |
||||
# additional_fields parameter used below, which allows for more fields beyond |
||||
# this default set. |
||||
BASE_ADDL_FIELDS = ['req_id', 'external_ctx', 'user'] |
||||
LOG = logging.getLogger(__name__) |
||||
|
||||
|
||||
def set_logvar(key, value): |
||||
""" Attempts to set the logvar in the request scope, or ignores it |
||||
if not running in an environment that supports it. |
||||
""" |
||||
if value: |
||||
if uwsgi: |
||||
uwsgi.set_logvar(key, value) |
||||
# If there were a different handler than uwsgi, here is where we'd |
||||
# need to set the logvar value for its use. |
||||
|
||||
|
||||
def assign_request_filter(handler, additional_fields=None): |
||||
"""Adds the request-scoped filter log filter to the passed handler |
||||
|
||||
:param handler: a logging handler, e.g. a ConsoleHandler |
||||
:param additional_fields: fields that will be included in the logging |
||||
records (if a matching logging format is used) |
||||
""" |
||||
handler_cls = handler.__class__.__name__ |
||||
if uwsgi: |
||||
if additional_fields is None: |
||||
additional_fields = [] |
||||
addl_fields = [*BASE_ADDL_FIELDS, *additional_fields] |
||||
handler.addFilter(UwsgiLogFilter(uwsgi, addl_fields)) |
||||
LOG.info("UWSGI present, added UWSGI log filter to handler %s", |
||||
handler_cls) |
||||
# if there are other handlers that would allow for request scoped logging |
||||
# to be set up, we could include those options here. |
||||
else: |
||||
LOG.info("No request based logging filter in the current environment. " |
||||
"No log filter added to handler %s", handler_cls) |
||||
@ -0,0 +1,76 @@
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""A UWSGI-dependent implementation of a logging filter allowing for |
||||
request-based logging. |
||||
""" |
||||
import logging |
||||
import threading |
||||
|
||||
|
||||
class UwsgiLogFilter(logging.Filter): |
||||
""" A filter that preepends log records with additional request |
||||
based information, or information provided by log_extra in the |
||||
kwargs provided to a thread |
||||
""" |
||||
def __init__(self, uwsgi, additional_fields=None): |
||||
super().__init__() |
||||
if additional_fields is None: |
||||
additional_fields = [] |
||||
self.uwsgi = uwsgi |
||||
self.log_fields = additional_fields |
||||
|
||||
def filter(self, record): |
||||
""" Checks for thread provided values, or attempts to get values |
||||
from uwsgi |
||||
""" |
||||
if self._thread_has_log_extra(): |
||||
value_setter = self._set_values_from_log_extra |
||||
else: |
||||
value_setter = self._set_value |
||||
|
||||
for field_nm in self.log_fields: |
||||
value_setter(record, field_nm) |
||||
return True |
||||
|
||||
def _set_value(self, record, logvar): |
||||
# handles setting the logvars from uwsgi or '' in case of none/empty |
||||
try: |
||||
logvar_value = self.uwsgi.get_logvar(logvar) |
||||
if logvar_value: |
||||
setattr(record, logvar, logvar_value.decode('UTF-8')) |
||||
else: |
||||
setattr(record, logvar, '') |
||||
except SystemError: |
||||
# This happens if log_extra is not on a thread that is spawned |
||||
# by a process running under uwsgi |
||||
setattr(record, logvar, '') |
||||
|
||||
def _set_values_from_log_extra(self, record, logvar): |
||||
# sets the values from the log_extra on the thread |
||||
setattr(record, logvar, self._get_value_from_thread(logvar) or '') |
||||
|
||||
def _thread_has_log_extra(self): |
||||
# Checks to see if log_extra is present on the current thread |
||||
if self._get_log_extra_from_thread(): |
||||
return True |
||||
return False |
||||
|
||||
def _get_value_from_thread(self, logvar): |
||||
# retrieve the logvar from the log_extra from kwargs for the thread |
||||
return self._get_log_extra_from_thread().get(logvar, '') |
||||
|
||||
def _get_log_extra_from_thread(self): |
||||
# retrieves the log_extra value from kwargs or {} if it doesn't |
||||
# exist |
||||
return threading.current_thread()._kwargs.get('log_extra', {}) |
||||
@ -1,147 +0,0 @@
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
""" A logging filter to prepend UWSGI-handled formatting to all logging |
||||
records that use this filter. Request-based values will cause the log |
||||
records to have correlation values that can be used to better trace |
||||
logs. If uwsgi is not present, does not attempt to change the logs in |
||||
any way |
||||
|
||||
Threads initiated using threading.Thread can be correlated to the request |
||||
they came from by setting a kwarg of log_extra, containing a dictionary |
||||
of valeus matching the VALID_ADDL_FIELDS below and any fields that are set |
||||
as additional_fields by the setup_logging function. This mechanism assumes |
||||
that the thread will maintain the correlation values for the life |
||||
of the thread. |
||||
""" |
||||
import logging |
||||
import threading |
||||
|
||||
# Import uwsgi to determine if it has been provided to the application. |
||||
try: |
||||
import uwsgi |
||||
except ImportError: |
||||
uwsgi = None |
||||
|
||||
VALID_ADDL_FIELDS = ['req_id', 'external_ctx', 'user'] |
||||
_DEFAULT_LOG_FORMAT = ( |
||||
'%(asctime)s %(levelname)-8s %(req_id)s %(external_ctx)s %(user)s ' |
||||
'%(module)s(%(lineno)d) %(funcName)s - %(message)s' |
||||
) |
||||
|
||||
_LOG_FORMAT_IN_USE = None |
||||
|
||||
|
||||
def setup_logging(level, format_string=None, additional_fields=None): |
||||
""" Establishes the base logging using the appropriate filter |
||||
attached to the console/stream handler. |
||||
:param level: The level value to set as the threshold for |
||||
logging. Ideally a client would use logging.INFO or |
||||
the desired logging constant to set this level value |
||||
:param format_string: Optional value allowing for override of the |
||||
logging format string. If new values beyond |
||||
the default value are introduced, the |
||||
additional_fields must contain those fields |
||||
to ensure they are set upon using the logging |
||||
filter. |
||||
:param additional_fields: Optionally allows for specifying more |
||||
fields that will be set on each logging |
||||
record. If specified, the format_string |
||||
parameter should be set with matching |
||||
fields, otherwise they will not be |
||||
displayed. |
||||
""" |
||||
global _LOG_FORMAT_IN_USE |
||||
_LOG_FORMAT_IN_USE = format_string or _DEFAULT_LOG_FORMAT |
||||
|
||||
console_handler = logging.StreamHandler() |
||||
if uwsgi: |
||||
console_handler.addFilter(UwsgiLogFilter(additional_fields)) |
||||
logging.basicConfig(level=level, |
||||
format=_LOG_FORMAT_IN_USE, |
||||
handlers=[console_handler]) |
||||
logger = logging.getLogger(__name__) |
||||
logger.info('Established logging defaults') |
||||
|
||||
|
||||
def get_log_format(): |
||||
""" Returns the common log format being used by this application |
||||
""" |
||||
return _LOG_FORMAT_IN_USE |
||||
|
||||
|
||||
def set_logvar(key, value): |
||||
""" Attempts to set the logvar in the request scope , or ignores it |
||||
if not running in uwsgi |
||||
""" |
||||
if uwsgi and value: |
||||
uwsgi.set_logvar(key, value) |
||||
|
||||
|
||||
class UwsgiLogFilter(logging.Filter): |
||||
""" A filter that preepends log records with additional request |
||||
based information, or information provided by log_extra in the |
||||
kwargs provided to a thread |
||||
""" |
||||
def __init__(self, additional_fields=None): |
||||
super().__init__() |
||||
if additional_fields is None: |
||||
additional_fields = [] |
||||
self.log_fields = [*VALID_ADDL_FIELDS, *additional_fields] |
||||
|
||||
def filter(self, record): |
||||
""" Checks for thread provided values, or attempts to get values |
||||
from uwsgi |
||||
""" |
||||
if self._thread_has_log_extra(): |
||||
value_setter = self._set_values_from_log_extra |
||||
else: |
||||
value_setter = self._set_value |
||||
|
||||
for field_nm in self.log_fields: |
||||
value_setter(record, field_nm) |
||||
return True |
||||
|
||||
def _set_value(self, record, logvar): |
||||
# handles setting the logvars from uwsgi or '' in case of none/empty |
||||
try: |
||||
logvar_value = None |
||||
if uwsgi: |
||||
logvar_value = uwsgi.get_logvar(logvar) |
||||
if logvar_value: |
||||
setattr(record, logvar, logvar_value.decode('UTF-8')) |
||||
else: |
||||
setattr(record, logvar, '') |
||||
except SystemError: |
||||
# This happens if log_extra is not on a thread that is spawned |
||||
# by a process running under uwsgi |
||||
setattr(record, logvar, '') |
||||
|
||||
def _set_values_from_log_extra(self, record, logvar): |
||||
# sets the values from the log_extra on the thread |
||||
setattr(record, logvar, self._get_value_from_thread(logvar) or '') |
||||
|
||||
def _thread_has_log_extra(self): |
||||
# Checks to see if log_extra is present on the current thread |
||||
if self._get_log_extra_from_thread(): |
||||
return True |
||||
return False |
||||
|
||||
def _get_value_from_thread(self, logvar): |
||||
# retrieve the logvar from the log_extra from kwargs for the thread |
||||
return self._get_log_extra_from_thread().get(logvar, '') |
||||
|
||||
def _get_log_extra_from_thread(self): |
||||
# retrieves the log_extra value from kwargs or {} if it doesn't |
||||
# exist |
||||
return threading.current_thread()._kwargs.get('log_extra', {}) |
||||
@ -0,0 +1,119 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""String redaction based upon regex patterns |
||||
|
||||
Uses the keys and patterns from oslo_utils as a starting point, and layers in |
||||
new keys and patterns defined locally. |
||||
""" |
||||
import re |
||||
|
||||
# oslo_utils makes the choice to keep the list of supported redactions |
||||
# non-extensible (purposefully). Here we can define new values, and follow the |
||||
# same basic logic as used in strutils. |
||||
# The following are copied from oslo_utils, strutils. Extend using the other |
||||
# facilties to make these easier to keep in sync: |
||||
_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+'] |
||||
_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\'])[^\"\']*([\"\'])', |
||||
r'(%(key)s\s+[\"\'])[^\"\']*([\"\'])', |
||||
r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)', |
||||
r'(<%(key)s>)[^<]*(</%(key)s>)', |
||||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\'])[^\"\']*([\"\'])', |
||||
r'([\'"][^"\']*%(key)s[\'"]\s*:\s*u?[\'"])[^\"\']*' |
||||
'([\'"])', |
||||
r'([\'"][^\'"]*%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?' |
||||
'[\'"])[^\"\']*([\'"])', |
||||
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] |
||||
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password', |
||||
'auth_token', 'new_pass', 'auth_password', 'secret_uuid', |
||||
'secret', 'sys_pswd', 'token', 'configdrive', |
||||
'CHAPPASSWORD', 'encrypted_key'] |
||||
|
||||
|
||||
class Redactor(): |
||||
"""String redactor that can be used to replace sensitive values |
||||
|
||||
:param redaction: the string value to put in place in case of a redaction |
||||
:param keys: list of additional keys that are searched for and have related |
||||
values redacted |
||||
:param single_patterns: list of additional single capture group regex |
||||
patterns |
||||
:param double_patterns: list of additional double capture group regex |
||||
patterns |
||||
""" |
||||
# Start with the values defined in strutils |
||||
_KEYS = list(_SANITIZE_KEYS) |
||||
|
||||
_SINGLE_CG_PATTERNS = list(_FORMAT_PATTERNS_1) |
||||
_DOUBLE_CG_PATTERNS = list(_FORMAT_PATTERNS_2) |
||||
|
||||
# More keys to extend the set of keys used in identifying redactions |
||||
_KEYS.extend([]) |
||||
# More single capture group patterns |
||||
_SINGLE_CG_PATTERNS.extend([r'(%(key)s\s*[:]\s*)[^\s^\'^\"]+']) |
||||
# More two capture group patterns |
||||
_DOUBLE_CG_PATTERNS.extend([]) |
||||
|
||||
def __init__(self, |
||||
redaction='***', |
||||
keys=None, |
||||
single_patterns=None, |
||||
double_patterns=None): |
||||
if keys is None: |
||||
keys = [] |
||||
if single_patterns is None: |
||||
single_patterns = [] |
||||
if double_patterns is None: |
||||
double_patterns = [] |
||||
|
||||
self.redaction = redaction |
||||
|
||||
self.keys = list(Redactor._KEYS) |
||||
self.keys.extend(keys) |
||||
|
||||
singles = list(Redactor._SINGLE_CG_PATTERNS) |
||||
singles.extend(single_patterns) |
||||
|
||||
doubles = list(Redactor._DOUBLE_CG_PATTERNS) |
||||
doubles.extend(double_patterns) |
||||
|
||||
self._single_cg_patterns = self._gen_patterns(patterns=singles) |
||||
self._double_cg_patterns = self._gen_patterns(patterns=doubles) |
||||
# the two capture group patterns |
||||
|
||||
def _gen_patterns(self, patterns): |
||||
"""Initialize the redaction patterns""" |
||||
regex_patterns = {} |
||||
for key in self.keys: |
||||
regex_patterns[key] = [] |
||||
for pattern in patterns: |
||||
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) |
||||
regex_patterns[key].append(reg_ex) |
||||
return regex_patterns |
||||
|
||||
def redact(self, message): |
||||
"""Apply regex based replacements to mask values |
||||
|
||||
Modeled from: |
||||
https://github.com/openstack/oslo.utils/blob/9b23c17a6be6d07d171b64ada3629c3680598f7b/oslo_utils/strutils.py#L272 |
||||
""" |
||||
substitute1 = r'\g<1>' + self.redaction |
||||
substitute2 = r'\g<1>' + self.redaction + r'\g<2>' |
||||
|
||||
for key in self.keys: |
||||
if key in message: |
||||
for pattern in self._double_cg_patterns[key]: |
||||
message = re.sub(pattern, substitute2, message) |
||||
for pattern in self._single_cg_patterns[key]: |
||||
message = re.sub(pattern, substitute1, message) |
||||
return message |
||||
@ -0,0 +1,106 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Tests for redaction functionality |
||||
|
||||
- redaction_formatter |
||||
- redactor |
||||
""" |
||||
import logging |
||||
import sys |
||||
|
||||
from shipyard_airflow.control.logging.redaction_formatter import ( |
||||
RedactionFormatter |
||||
) |
||||
from shipyard_airflow.control.util.redactor import Redactor |
||||
|
||||
|
||||
class TestRedaction(): |
||||
|
||||
def test_redactor(self): |
||||
redactor = Redactor() |
||||
|
||||
to_redact = "My password = swordfish" |
||||
expected = "My password = ***" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
# not a detected pattern - should be same |
||||
to_redact = "My pass = swordfish" |
||||
expected = "My pass = swordfish" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
# yaml format |
||||
to_redact = "My password: swordfish" |
||||
expected = "My password: ***" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
# yaml format |
||||
to_redact = "My password: swordfish is not for sale" |
||||
expected = "My password: *** is not for sale" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
to_redact = """ |
||||
password: |
||||
swordfish |
||||
""" |
||||
expected = """ |
||||
password: |
||||
*** |
||||
""" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
to_redact = """ |
||||
password: |
||||
swordfish |
||||
trains: |
||||
cheese |
||||
""" |
||||
expected = """ |
||||
password: |
||||
*** |
||||
trains: |
||||
cheese |
||||
""" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
def test_extended_keys_redactor(self): |
||||
redactor = Redactor(redaction="++++", keys=['trains']) |
||||
to_redact = """ |
||||
password: |
||||
swordfish |
||||
trains: |
||||
cheese |
||||
""" |
||||
expected = """ |
||||
password: |
||||
++++ |
||||
trains: |
||||
++++ |
||||
""" |
||||
assert redactor.redact(to_redact) == expected |
||||
|
||||
def test_redaction_formatter(self, caplog): |
||||
# since root logging is setup by prior tests need to remove all |
||||
# handlers to simulate a clean environment of setting up this |
||||
# RedactionFormatter |
||||
for handler in list(logging.root.handlers): |
||||
if not "LogCaptureHandler" == handler.__class__.__name__: |
||||
logging.root.removeHandler(handler) |
||||
|
||||
logging.basicConfig(level=logging.DEBUG, |
||||
handlers=[logging.StreamHandler(sys.stdout)]) |
||||
for handler in logging.root.handlers: |
||||
handler.setFormatter(RedactionFormatter(handler.formatter)) |
||||
logging.info('Established password: albatross for this test') |
||||
assert 'albatross' not in caplog.text |
||||
assert 'Established password: *** for this test' in caplog.text |
||||
Loading…
Reference in new issue