6d1349eb8b
This commit includes two cleanups in trove-pylint.
First, the deprecation warning is because the text.ParseableOutputFormat is
being deprecated[1] and produces a deprecation warning as shown in the
bug.
Second, while looking into 1650816, I stumbled upon 1650888 which is
also fixed. A block of code was wrongly indented and when I disabled
"-E" to generate warnings, produced a crash in pylint. If the line
generated doesn't match the regular expression, the various parameters
to the next call (to ignore()) are not set and you get unbound
variable errors.
[1] https://github.com/PyCQA/pylint/blob/master/pylint/reporters/text.py#L163
Change-Id: I43bd6569a98eeb1b326e4a0ca1addaf97c0830a5
Closes-Bug: #1650530
Closes-But: #1650888
353 lines
11 KiB
Python
Executable File
353 lines
11 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2016 Tesora, Inc.
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import print_function
|
|
|
|
import fnmatch
|
|
import json
|
|
from collections import OrderedDict
|
|
import os
|
|
import re
|
|
import six
|
|
import sys
|
|
|
|
from pylint import lint
|
|
from pylint.reporters import text
|
|
from six.moves import cStringIO as csio
|
|
|
|
DEFAULT_CONFIG_FILE = "tools/trove-pylint.config"
|
|
DEFAULT_IGNORED_FILES = ['trove/tests']
|
|
DEFAULT_IGNORED_CODES = []
|
|
DEFAULT_IGNORED_MESSAGES = []
|
|
DEFAULT_ALWAYS_ERROR = [
|
|
"Undefined variable '_'",
|
|
"Undefined variable '_LE'",
|
|
"Undefined variable '_LI'",
|
|
"Undefined variable '_LW'",
|
|
"Undefined variable '_LC'"]
|
|
|
|
MODE_CHECK = "check"
|
|
MODE_REBUILD = "rebuild"
|
|
|
|
class Config(object):
|
|
def __init__(self, filename=DEFAULT_CONFIG_FILE):
|
|
|
|
self.default_config = {
|
|
"include": ["*.py"],
|
|
"folder": "trove",
|
|
"options": ["--rcfile=./pylintrc", "-E"],
|
|
"ignored_files": DEFAULT_IGNORED_FILES,
|
|
"ignored_codes": DEFAULT_IGNORED_CODES,
|
|
"ignored_messages": DEFAULT_IGNORED_MESSAGES,
|
|
"ignored_file_codes": [],
|
|
"ignored_file_messages": [],
|
|
"ignored_file_code_messages": [],
|
|
"always_error_messages": DEFAULT_ALWAYS_ERROR
|
|
}
|
|
|
|
self.config = self.default_config
|
|
|
|
def sort_config(self):
|
|
sorted_config = OrderedDict()
|
|
for key in sorted(self.config.keys()):
|
|
value = self.get(key)
|
|
if isinstance(value, list) and not isinstance(value,
|
|
six.string_types):
|
|
sorted_config[key] = sorted(value)
|
|
else:
|
|
sorted_config[key] = value
|
|
|
|
return sorted_config
|
|
|
|
def save(self, filename=DEFAULT_CONFIG_FILE):
|
|
if os.path.isfile(filename):
|
|
os.rename(filename, "%s~" % filename)
|
|
|
|
with open(filename, 'w') as fp:
|
|
json.dump(self.sort_config(), fp, encoding="utf-8",
|
|
indent=2, separators=(',', ': '))
|
|
|
|
def load(self, filename=DEFAULT_CONFIG_FILE):
|
|
with open(filename) as fp:
|
|
self.config = json.load(fp, encoding="utf-8")
|
|
|
|
def get(self, attribute):
|
|
return self.config[attribute]
|
|
|
|
def is_file_ignored(self, f):
|
|
if any(f.startswith(i)
|
|
for i in self.config['ignored_files']):
|
|
return True
|
|
|
|
return False
|
|
|
|
def is_file_included(self, f):
|
|
if any(fnmatch.fnmatch(f, wc) for wc in self.config['include']):
|
|
return True
|
|
|
|
return False
|
|
|
|
def is_always_error(self, message):
|
|
if message in self.config['always_error_messages']:
|
|
return True
|
|
|
|
return False
|
|
|
|
def ignore(self, filename, code, codename, message):
|
|
# the high priority checks
|
|
if self.is_file_ignored(filename):
|
|
return True
|
|
|
|
# never ignore messages
|
|
if self.is_always_error(message):
|
|
return False
|
|
|
|
if code in self.config['ignored_codes']:
|
|
return True
|
|
|
|
if codename in self.config['ignored_codes']:
|
|
return True
|
|
|
|
if message and any(message.startswith(ignore_message)
|
|
for ignore_message
|
|
in self.config['ignored_messages']):
|
|
return True
|
|
|
|
if filename and message and (
|
|
[filename, message] in self.config['ignored_file_messages']):
|
|
return True
|
|
|
|
if filename and code and (
|
|
[filename, code] in self.config['ignored_file_codes']):
|
|
return True
|
|
|
|
if filename and codename and (
|
|
[filename, codename] in self.config['ignored_file_codes']):
|
|
return True
|
|
|
|
fcm_ignore1 = [filename, code, message]
|
|
fcm_ignore2 = [filename, codename, message]
|
|
for fcm in self.config['ignored_file_code_messages']:
|
|
if fcm_ignore1 == [fcm[0], fcm[1], fcm[2]]:
|
|
return True
|
|
|
|
if fcm_ignore2 == [fcm[0], fcm[1], fcm[2]]:
|
|
return True
|
|
|
|
return False
|
|
|
|
def ignore_code(self, c):
|
|
_c = set(self.config['ignored_codes'])
|
|
_c.add(c)
|
|
self.config['ignored_codes'] = list(_c)
|
|
|
|
def ignore_files(self, f):
|
|
_c = set(self.config['ignored_files'])
|
|
_c.add(f)
|
|
self.config['ignored_files'] = list(_c)
|
|
|
|
def ignore_message(self, m):
|
|
_c = set(self.config['ignored_messages'])
|
|
_c.add(m)
|
|
self.config['ignored_messages'] = list(_c)
|
|
|
|
def ignore_file_code(self, f, c):
|
|
_c = set(self.config['ignored_file_codes'])
|
|
_c.add((f, c))
|
|
self.config['ignored_file_codes'] = list(_c)
|
|
|
|
def ignore_file_message(self, f, m):
|
|
_c = set(self.config['ignored_file_messages'])
|
|
_c.add((f, m))
|
|
self.config['ignored_file_messages'] = list(_c)
|
|
|
|
def ignore_file_code_message(self, f, c, m, fn):
|
|
_c = set(self.config['ignored_file_code_messages'])
|
|
_c.add((f, c, m, fn))
|
|
self.config['ignored_file_code_messages'] = list(_c)
|
|
|
|
def main():
|
|
if len(sys.argv) == 1 or sys.argv[1] == "check":
|
|
return check()
|
|
elif sys.argv[1] == "rebuild":
|
|
return rebuild()
|
|
elif sys.argv[1] == "initialize":
|
|
return initialize()
|
|
else:
|
|
return usage()
|
|
|
|
def usage():
|
|
print("Usage: %s [check|rebuild]" % sys.argv[0])
|
|
print("\tUse this tool to perform a lint check of the trove project.")
|
|
print("\t check: perform the lint check.")
|
|
print("\t rebuild: rebuild the list of exceptions to ignore.")
|
|
return 0
|
|
|
|
class ParseableTextReporter(text.TextReporter):
|
|
name = 'parseable'
|
|
line_format = '{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'
|
|
|
|
# that's it folks
|
|
|
|
|
|
class LintRunner(object):
|
|
def __init__(self):
|
|
self.config = Config()
|
|
self.idline = re.compile("^[*]* Module .*")
|
|
self.detail = re.compile("(\S+):(\d+): \[(\S+)\((\S+)\), (\S+)?] (.*)")
|
|
|
|
def dolint(self, filename):
|
|
exceptions = set()
|
|
|
|
buffer = csio()
|
|
reporter = ParseableTextReporter(output=buffer)
|
|
options = list(self.config.get('options'))
|
|
options.append(filename)
|
|
lint.Run(options, reporter=reporter, exit=False)
|
|
|
|
output = buffer.getvalue()
|
|
buffer.close()
|
|
|
|
for line in output.splitlines():
|
|
if self.idline.match(line):
|
|
continue
|
|
|
|
if self.detail.match(line):
|
|
mo = self.detail.search(line)
|
|
tokens = mo.groups()
|
|
fn = tokens[0]
|
|
ln = tokens[1]
|
|
code = tokens[2]
|
|
codename = tokens[3]
|
|
func = tokens[4]
|
|
message = tokens[5]
|
|
|
|
if not self.config.ignore(fn, code, codename, message):
|
|
exceptions.add((fn, ln, code, codename, func, message))
|
|
|
|
return exceptions
|
|
|
|
def process(self, mode=MODE_CHECK):
|
|
files_processed = 0
|
|
files_with_errors = 0
|
|
errors_recorded = 0
|
|
exceptions_recorded = 0
|
|
all_exceptions = []
|
|
|
|
for (root, dirs, files) in os.walk(self.config.get('folder')):
|
|
# if we shouldn't even bother about this part of the
|
|
# directory structure, we can punt quietly
|
|
if self.config.is_file_ignored(root):
|
|
continue
|
|
|
|
# since we are walking top down, let's clean up the dirs
|
|
# that we will walk by eliminating any dirs that will
|
|
# end up getting ignored
|
|
for d in dirs:
|
|
p = os.path.join(root, d)
|
|
if self.config.is_file_ignored(p):
|
|
dirs.remove(d)
|
|
|
|
# check if we can ignore the file and process if not
|
|
for f in files:
|
|
p = os.path.join(root, f)
|
|
if self.config.is_file_ignored(p):
|
|
continue
|
|
|
|
if not self.config.is_file_included(f):
|
|
continue
|
|
|
|
files_processed += 1
|
|
exceptions = self.dolint(p)
|
|
file_had_errors = 0
|
|
|
|
for e in exceptions:
|
|
# what we do with this exception depents on the
|
|
# kind of exception, and the mode
|
|
if self.config.is_always_error(e[5]):
|
|
all_exceptions.append(e)
|
|
errors_recorded += 1
|
|
file_had_errors += 1
|
|
elif mode == MODE_REBUILD:
|
|
# parameters to ignore_file_code_message are
|
|
# filename, code, message and function
|
|
self.config.ignore_file_code_message(e[0], e[2], e[-1], e[4])
|
|
self.config.ignore_file_code_message(e[0], e[3], e[-1], e[4])
|
|
exceptions_recorded += 1
|
|
elif mode == MODE_CHECK:
|
|
all_exceptions.append(e)
|
|
errors_recorded += 1
|
|
file_had_errors += 1
|
|
|
|
if file_had_errors:
|
|
files_with_errors += 1
|
|
|
|
for e in sorted(all_exceptions):
|
|
print("ERROR: %s %s: %s %s, %s: %s" %
|
|
(e[0], e[1], e[2], e[3], e[4], e[5]))
|
|
|
|
return (files_processed, files_with_errors, errors_recorded,
|
|
exceptions_recorded)
|
|
|
|
def rebuild(self):
|
|
self.initialize()
|
|
(files_processed,
|
|
files_with_errors,
|
|
errors_recorded,
|
|
exceptions_recorded) = self.process(mode=MODE_REBUILD)
|
|
|
|
if files_with_errors > 0:
|
|
print("Rebuild failed. %s files processed, %s had errors, "
|
|
"%s errors recorded." % (
|
|
files_processed, files_with_errors, errors_recorded))
|
|
|
|
return 1
|
|
|
|
self.config.save()
|
|
print("Rebuild completed. %s files processed, %s exceptions recorded." %
|
|
(files_processed, exceptions_recorded))
|
|
return 0
|
|
|
|
def check(self):
|
|
self.config.load()
|
|
(files_processed,
|
|
files_with_errors,
|
|
errors_recorded,
|
|
exceptions_recorded) = self.process(mode=MODE_CHECK)
|
|
|
|
if files_with_errors > 0:
|
|
print("Check failed. %s files processed, %s had errors, "
|
|
"%s errors recorded." % (
|
|
files_processed, files_with_errors, errors_recorded))
|
|
return 1
|
|
|
|
print("Check succeeded. %s files processed" % files_processed)
|
|
return 0
|
|
|
|
def initialize(self):
|
|
self.config.save()
|
|
return 0
|
|
|
|
def check():
|
|
exit(LintRunner().check())
|
|
|
|
def rebuild():
|
|
exit(LintRunner().rebuild())
|
|
|
|
def initialize():
|
|
exit(LintRunner().initialize())
|
|
|
|
if __name__ == "__main__":
|
|
main()
|