From d159335700938f25ebd2606c066e3895e2a3d577 Mon Sep 17 00:00:00 2001 From: Marek Cermak Date: Mon, 9 Oct 2017 15:57:56 +0200 Subject: [PATCH] Custom formatter Implements: custom formatter Custom formatter can be used to output a machine-readable, easily parsable and customizable format using set of predefined tags to suite various needs. Output string is formatted using python string.format() standards and therefore provides familiar usage. Usage: bandit --format custom [--msg-template MSG-TEMPLATE] targets See bandit --help for additional information and list of available tags modified: bandit/cli/main.py modified: bandit/core/manager.py modified: README.rst modified: setup.cfg new file: bandit/formatters/custom.py Change-Id: I900c9689cddb048db58608c443305e05e7a4be14 Signed-off-by: Marek Cermak --- README.rst | 64 +++++++++++- bandit/cli/main.py | 45 ++++++++- bandit/core/manager.py | 14 ++- bandit/formatters/custom.py | 163 +++++++++++++++++++++++++++++++ doc/source/man/bandit.rst | 35 ++++++- setup.cfg | 1 + tests/functional/test_runtime.py | 2 +- 7 files changed, 311 insertions(+), 13 deletions(-) create mode 100644 bandit/formatters/custom.py diff --git a/README.rst b/README.rst index 41aede1c..de3773f8 100644 --- a/README.rst +++ b/README.rst @@ -87,8 +87,9 @@ Usage:: $ bandit -h usage: bandit [-h] [-r] [-a {file,vuln}] [-n CONTEXT_LINES] [-c CONFIG_FILE] [-p PROFILE] [-t TESTS] [-s SKIPS] [-l] [-i] - [-f {csv,html,json,screen,txt,xml,yaml}] [-o [OUTPUT_FILE]] [-v] - [-d] [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE] + [-f {csv,custom,html,json,screen,txt,xml,yaml}] + [--msg-template MSG_TEMPLATE] [-o [OUTPUT_FILE]] [-v] [-d] + [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE] [--ini INI_PATH] [--version] targets [targets ...] @@ -118,8 +119,12 @@ Usage:: (-l for LOW, -ll for MEDIUM, -lll for HIGH) -i, --confidence report only issues of a given confidence level or higher (-i for LOW, -ii for MEDIUM, -iii for HIGH) - -f {csv,html,json,screen,txt,xml,yaml}, --format {csv,html,json,screen,txt,xml,yaml} + -f {csv,custom,html,json,screen,txt,xml,yaml}, --format {csv,custom,html,json,screen,txt,xml,yaml} specify output format + --msg-template MSG_TEMPLATE + specify output message template (only usable with + --format custom), see CUSTOM FORMAT section for list + of available values -o [OUTPUT_FILE], --output [OUTPUT_FILE] write report to filename -v, --verbose output extra information like excluded and included @@ -137,7 +142,33 @@ Usage:: arguments --version show program's version number and exit + CUSTOM FORMATTING + ----------------- + + Available tags: + + {abspath}, {relpath}, {line}, {test_id}, + {severity}, {msg}, {confidence}, {range} + + Example usage: + + Default template: + bandit -r examples/ --format custom --msg-template \ + "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}" + + Provides same output as: + bandit -r examples/ --format custom + + Tags can also be formatted in python string.format() style: + bandit -r examples/ --format custom --msg-template \ + "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}" + + See python documentation for more information about formatting style: + https://docs.python.org/3.4/library/string.html + The following tests were discovered and loaded: + ----------------------------------------------- + B101 assert_used B102 exec_used B103 set_bad_file_permissions @@ -339,6 +370,33 @@ To register your plugin, you have two options: bandit.plugins = mako = bandit_mako + +Custom Formatting +----------------- + +Available tags: + +:: + {abspath}, {relpath}, {line}, {test_id}, + {severity}, {msg}, {confidence}, {range} + +Example usage: + + Default template:: + bandit -r examples/ --format custom --msg-template \ + "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}" + + Provides same output as:: + bandit -r examples/ --format custom + + Tags can also be formatted in python string.format() style:: + bandit -r examples/ --format custom --msg-template \ + "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}" + +See python documentation for more information about formatting style: +https://docs.python.org/3.4/library/string.html + + Contributing ------------ Contributions to Bandit are always welcome! We can be found on diff --git a/bandit/cli/main.py b/bandit/cli/main.py index 423e95c7..2c4a4030 100644 --- a/bandit/cli/main.py +++ b/bandit/cli/main.py @@ -18,6 +18,7 @@ import fnmatch import logging import os import sys +import textwrap import bandit @@ -205,6 +206,13 @@ def main(): default=output_format, help='specify output format', choices=sorted(extension_mgr.formatter_names) ) + parser.add_argument( + '--msg-template', action='store', + default=None, help='specify output message template' + ' (only usable with --format custom),' + ' see CUSTOM FORMAT section' + ' for list of available values', + ) parser.add_argument( '-o', '--output', dest='output_file', action='store', nargs='?', type=argparse.FileType('w'), default=sys.stdout, @@ -253,11 +261,41 @@ def main(): blacklist_info.append('%s\t%s' % (b['id'], b['name'])) plugin_list = '\n\t'.join(sorted(set(plugin_info + blacklist_info))) - parser.epilog = ('The following tests were discovered and' - ' loaded:\n\t{0}\n'.format(plugin_list)) + dedent_text = textwrap.dedent(''' + CUSTOM FORMATTING + ----------------- + + Available tags: + + {abspath}, {relpath}, {line}, {test_id}, + {severity}, {msg}, {confidence}, {range} + + Example usage: + + Default template: + bandit -r examples/ --format custom --msg-template \\ + "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}" + + Provides same output as: + bandit -r examples/ --format custom + + Tags can also be formatted in python string.format() style: + bandit -r examples/ --format custom --msg-template \\ + "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}" + + See python documentation for more information about formatting style: + https://docs.python.org/3.4/library/string.html + + The following tests were discovered and loaded: + ----------------------------------------------- + ''') + parser.epilog = dedent_text + "\t{0}".format(plugin_list) # setup work - parse arguments, and initialize BanditManager args = parser.parse_args() + # Check if `--msg-template` is not present without custom formatter + if args.output_format != 'custom' and args.msg_template is not None: + parser.error("--msg-template can only be used with --format=custom") try: b_conf = b_config.BanditConfig(config_file=args.config_file) @@ -341,7 +379,8 @@ def main(): sev_level, conf_level, args.output_file, - args.output_format) + args.output_format, + args.msg_template) # return an exit code of 1 if there are results, 0 otherwise if b_mgr.results_count(sev_filter=sev_level, conf_filter=conf_level) > 0: diff --git a/bandit/core/manager.py b/bandit/core/manager.py index d4febced..cb8b574b 100644 --- a/bandit/core/manager.py +++ b/bandit/core/manager.py @@ -136,7 +136,7 @@ class BanditManager(object): return len(self.get_issue_list(sev_filter, conf_filter)) def output_results(self, lines, sev_level, conf_level, output_file, - output_format): + output_format, template=None): '''Outputs results from the result store :param lines: How many surrounding lines to show per result @@ -144,6 +144,9 @@ class BanditManager(object): :param conf_level: Which confidence levels to show (LOW, MEDIUM, HIGH) :param output_file: File to store results :param output_format: output format plugin name + :param template: Output template with non-terminal tags + (default: {abspath}:{line}: + {test_id}[bandit]: {severity}: {msg}) :return: - ''' try: @@ -153,8 +156,13 @@ class BanditManager(object): formatter = formatters_mgr[output_format] report_func = formatter.plugin - report_func(self, fileobj=output_file, sev_level=sev_level, - conf_level=conf_level, lines=lines) + if output_format == 'custom': + report_func(self, fileobj=output_file, sev_level=sev_level, + conf_level=conf_level, lines=lines, + template=template) + else: + report_func(self, fileobj=output_file, sev_level=sev_level, + conf_level=conf_level, lines=lines) except Exception as e: raise RuntimeError("Unable to output report using '%s' formatter: " diff --git a/bandit/formatters/custom.py b/bandit/formatters/custom.py new file mode 100644 index 00000000..864ff4f3 --- /dev/null +++ b/bandit/formatters/custom.py @@ -0,0 +1,163 @@ +# Copyright (c) 2017 Hewlett Packard Enterprise +# -*- coding:utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +r""" +================ +Custom Formatter +================ + +This formatter outputs the issues in custom machine-readable format. + +default template: {abspath}:{line}: {test_id}[bandit]: {severity}: {msg} + +:Example: + +/usr/lib/python3.6/site-packages/openlp/core/utils/__init__.py: \ +405: B310[bandit]: MEDIUM: Audit url open for permitted schemes. \ +Allowing use of file:/ or custom schemes is often unexpected. + +""" + +import logging +import os +import re +import string +import sys + +from bandit.core import test_properties + + +LOG = logging.getLogger(__name__) + + +class SafeMapper(dict): + """Safe mapper to handle format key errors""" + @classmethod # To prevent PEP8 warnings in the test suite + def __missing__(cls, key): + return "{%s}" % key + + +@test_properties.accepts_baseline +def report(manager, fileobj, sev_level, conf_level, lines=-1, template=None): + """Prints issues in custom format + + :param manager: the bandit manager object + :param fileobj: The output file object, which may be sys.stdout + :param sev_level: Filtering severity level + :param conf_level: Filtering confidence level + :param lines: Number of lines to report, -1 for all + :param template: Output template with non-terminal tags + (default: '{abspath}:{line}: + {test_id}[bandit]: {severity}: {msg}') + """ + + machine_output = {'results': [], 'errors': []} + for (fname, reason) in manager.get_skipped(): + machine_output['errors'].append({'filename': fname, + 'reason': reason}) + + results = manager.get_issue_list(sev_level=sev_level, + conf_level=conf_level) + + msg_template = template + if template is None: + msg_template = "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}" + + # Dictionary of non-terminal tags that will be expanded + tag_mapper = { + 'abspath': lambda issue: os.path.abspath(issue.fname), + 'relpath': lambda issue: os.path.relpath(issue.fname), + 'line': lambda issue: issue.lineno, + 'test_id': lambda issue: issue.test_id, + 'severity': lambda issue: issue.severity, + 'msg': lambda issue: issue.text, + 'confidence': lambda issue: issue.confidence, + 'range': lambda issue: issue.linerange + } + + # Create dictionary with tag sets to speed up search for similar tags + tag_sim_dict = dict( + [(tag, set(tag)) for tag, _ in tag_mapper.items()] + ) + + # Parse the format_string template and check the validity of tags + try: + parsed_template_orig = list(string.Formatter().parse(msg_template)) + # of type (literal_text, field_name, fmt_spec, conversion) + + # Check the format validity only, ignore keys + string.Formatter().vformat(msg_template, (), SafeMapper(line=0)) + except ValueError as e: + LOG.error("Template is not in valid format: %s", e.args[0]) + sys.exit(2) + + tag_set = {t[1] for t in parsed_template_orig if t[1] is not None} + if not tag_set: + LOG.error("No tags were found in the template. Are you missing '{}'?") + sys.exit(2) + + def get_similar_tag(tag): + similarity_list = [(len(set(tag) & t_set), t) + for t, t_set in tag_sim_dict.items()] + return sorted(similarity_list)[-1][1] + + tag_blacklist = [] + for tag in tag_set: + # check if the tag is in dictionary + if tag not in tag_mapper: + similar_tag = get_similar_tag(tag) + LOG.warning( + "Tag '%s' was not recognized and will be skipped, " + "did you mean to use '%s'?", tag, similar_tag + ) + tag_blacklist += [tag] + + # Compose the message template back with the valid values only + msg_parsed_template_list = [] + for literal_text, field_name, fmt_spec, conversion in parsed_template_orig: + if literal_text: + # if there is '{' or '}', double it to prevent expansion + literal_text = re.sub('{', '{{', literal_text) + literal_text = re.sub('}', '}}', literal_text) + msg_parsed_template_list.append(literal_text) + + if field_name is not None: + if field_name in tag_blacklist: + msg_parsed_template_list.append(field_name) + continue + # Append the fmt_spec part + params = [field_name, fmt_spec, conversion] + markers = ['', ':', '!'] + msg_parsed_template_list.append( + ['{'] + + ["%s" % (m + p) if p else '' + for m, p in zip(markers, params)] + + ['}'] + ) + + msg_parsed_template = "".join([item for lst in msg_parsed_template_list + for item in lst]) + "\n" + limit = lines if lines > 0 else None + with fileobj: + for defect in results[:limit]: + evaluated_tags = SafeMapper( + (k, v(defect)) for k, v in tag_mapper.items() + ) + output = msg_parsed_template.format(**evaluated_tags) + + fileobj.write(output) + + if fileobj.name != sys.stdout.name: + LOG.info("Result written to file: %s", fileobj.name) diff --git a/doc/source/man/bandit.rst b/doc/source/man/bandit.rst index 04a3a435..363b8575 100644 --- a/doc/source/man/bandit.rst +++ b/doc/source/man/bandit.rst @@ -7,8 +7,9 @@ SYNOPSIS bandit [-h] [-r] [-a {file,vuln}] [-n CONTEXT_LINES] [-c CONFIG_FILE] [-p PROFILE] [-t TESTS] [-s SKIPS] [-l] [-i] - [-f {csv,html,json,screen,txt,xml,yaml}] [-o OUTPUT_FILE] [-v] - [-d] [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE] + [-f {csv,custom,html,json,screen,txt,xml,yaml}] + [--msg-template MSG_TEMPLATE] [-o OUTPUT_FILE] [-v] [-d] + [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE] [--ini INI_PATH] [--version] targets [targets ...] @@ -43,8 +44,12 @@ OPTIONS (-l for LOW, -ll for MEDIUM, -lll for HIGH) -i, --confidence report only issues of a given confidence level or higher (-i for LOW, -ii for MEDIUM, -iii for HIGH) - -f {csv,html,json,screen,txt,xml,yaml}, --format {csv,html,json,screen,txt,xml,yaml} + -f {csv,custom,html,json,screen,txt,xml,yaml}, --format {csv,custom,html,json,screen,txt,xml,yaml} specify output format + --msg-template MSG_TEMPLATE + specify output message template (only usable with + --format custom), see CUSTOM FORMAT section for list + of available values -o OUTPUT_FILE, --output OUTPUT_FILE write report to filename -v, --verbose output extra information like excluded and included @@ -62,6 +67,30 @@ OPTIONS arguments --version show program's version number and exit +CUSTOM FORMATTING +----------------- + +Available tags: + + {abspath}, {relpath}, {line}, {test_id}, + {severity}, {msg}, {confidence}, {range} + +Example usage: + + Default template: + bandit -r examples/ --format custom --msg-template \ + "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}" + + Provides same output as: + bandit -r examples/ --format custom + + Tags can also be formatted in python string.format() style: + bandit -r examples/ --format custom --msg-template \ + "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}" + + See python documentation for more information about formatting style: + https://docs.python.org/3.4/library/string.html + FILES ===== diff --git a/setup.cfg b/setup.cfg index cb3aad64..78bcb8e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ bandit.formatters = html = bandit.formatters.html:report screen = bandit.formatters.screen:report yaml = bandit.formatters.yaml:report + custom = bandit.formatters.custom:report bandit.plugins = # bandit/plugins/app_debug.py flask_debug_true = bandit.plugins.app_debug:flask_debug_true diff --git a/tests/functional/test_runtime.py b/tests/functional/test_runtime.py index 2fe8ff29..5fa19977 100644 --- a/tests/functional/test_runtime.py +++ b/tests/functional/test_runtime.py @@ -77,7 +77,7 @@ class RuntimeTests(testtools.TestCase): self.assertIn("tests were discovered and loaded:", output) def test_help_in_readme(self): - replace_list = [' ', '\t'] + replace_list = [' ', '\t', '\n'] (retcode, output) = self._test_runtime(['bandit', '-h']) for i in replace_list: output = output.replace(i, '')