Browse Source

Simplify running pylint

This commit does several things:

- Setup and run pylint directly rather than running through a script.
  This allows the user to see what is happening while the user is running
  through pylint.
- Allow the user to either run pylint on a particular changeset,
  or the entire manila tree.
- Allow the user to run on a particular changeset. Using like HEAD~1,
  etc.
- I disabled the tests that were reported by pylint.
  The thought here would be go through the failures
  and correct them.
- Update pylint to 2.1.1.

Change-Id: I398d1ba01a3fd7f1c86ad4065a7984d276f33383
Signed-off-by: Chuck Short <chucks@redhat.com>
Chuck Short 6 months ago
parent
commit
4d0abb4a14
5 changed files with 247 additions and 293 deletions
  1. 189
    0
      .pylintrc
  2. 56
    0
      tools/coding-checks.sh
  3. 0
    232
      tools/lintstack.py
  4. 0
    59
      tools/lintstack.sh
  5. 2
    2
      tox.ini

+ 189
- 0
.pylintrc View File

@@ -0,0 +1,189 @@
1
+[MASTER]
2
+
3
+# A comma-separated list of package or module names from where C extensions may
4
+# be loaded. Extensions are loading into the active Python interpreter and may
5
+# run arbitrary code.
6
+extension-pkg-whitelist=
7
+
8
+# Add files or directories to the blacklist. They should be base names, not
9
+# paths.
10
+ignore=CVS,tests,test
11
+
12
+# Add files or directories matching the regex patterns to the blacklist. The
13
+# regex matches against base names, not paths.
14
+ignore-patterns=
15
+
16
+# Python code to execute, usually for sys.path manipulation such as
17
+# pygtk.require().
18
+#init-hook=
19
+
20
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
21
+# number of processors available to use.
22
+jobs=1
23
+
24
+# Control the amount of potential inferred values when inferring a single
25
+# object. This can help the performance when dealing with large functions or
26
+# complex, nested conditions.
27
+limit-inference-results=100
28
+
29
+# List of plugins (as comma separated values of python modules names) to load,
30
+# usually to register additional checkers.
31
+load-plugins=
32
+
33
+# Pickle collected data for later comparisons.
34
+persistent=yes
35
+
36
+# Specify a configuration file.
37
+#rcfile=
38
+
39
+# When enabled, pylint would attempt to guess common misconfiguration and emit
40
+# user-friendly hints instead of false-positive error messages.
41
+suggestion-mode=yes
42
+
43
+# Allow loading of arbitrary C extensions. Extensions are imported into the
44
+# active Python interpreter and may run arbitrary code.
45
+unsafe-load-any-extension=no
46
+
47
+
48
+[MESSAGES CONTROL]
49
+
50
+# Only show warnings with the listed confidence levels. Leave empty to show
51
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
52
+confidence=
53
+
54
+# Disable the message, report, category or checker with the given id(s). You
55
+# can either give multiple identifiers separated by comma (,) or put this
56
+# option multiple times (only on the command line, not in the configuration
57
+# file where it should appear only once). You can also use "--disable=all" to
58
+# disable everything first and then reenable specific checks. For example, if
59
+# you want to run only the similarities checker, you can use "--disable=all
60
+# --enable=similarities". If you want to run only the classes checker, but have
61
+# no Warning level messages displayed, use "--disable=all --enable=classes
62
+# --disable=W".
63
+disable=
64
+# "F" Fatal errors that prevent further processing
65
+ import-error,
66
+# "I" Informational noise
67
+ locally-disabled,
68
+ c-extension-no-member,
69
+# "E" Error for important programming issues (likely bugs)
70
+ no-member,
71
+ too-many-function-args,
72
+ not-callable,
73
+ assignment-from-none,
74
+ unsubscriptable-object,
75
+ used-prior-global-declaration,
76
+ not-an-iterable,
77
+# "W" Warnings for stylistic problems or minor programming issues
78
+ unused-argument,
79
+ bad-indentation,
80
+ unused-variable,
81
+ useless-else-on-loop,
82
+ pointless-string-statement,
83
+ unused-import,
84
+ redefined-outer-name,
85
+ redefined-builtin,
86
+ attribute-defined-outside-init,
87
+ abstract-method,
88
+ fixme,
89
+ exec-used,
90
+ anomalous-backslash-in-string,
91
+ broad-except,
92
+ protected-access,
93
+ arguments-differ,
94
+ undefined-loop-variable,
95
+ try-except-raise,
96
+ global-statement,
97
+ super-init-not-called,
98
+ pointless-statement,
99
+ global-statement,
100
+ unnecessary-lambda,
101
+ keyword-arg-before-vararg,
102
+ deprecated-method,
103
+ useless-super-delegation,
104
+ eval-used,
105
+ wildcard-import,
106
+ reimported,
107
+ expression-not-assigned,
108
+ cell-var-from-loop,
109
+ signature-differs,
110
+# "C" Coding convention violations
111
+ missing-docstring,
112
+ invalid-name,
113
+ wrong-import-order,
114
+ len-as-condition,
115
+ wrong-import-position,
116
+ bad-continuation,
117
+ too-many-lines,
118
+ misplaced-comparison-constant,
119
+ bad-mcs-classmethod-argument,
120
+ ungrouped-imports,
121
+ superfluous-parens,
122
+ unidiomatic-typecheck,
123
+ consider-iterating-dictionary,
124
+ bad-whitespace,
125
+ dangerous-default-value,
126
+ line-too-long,
127
+ consider-using-enumerate,
128
+ useless-import-alias,
129
+ singleton-comparison,
130
+# "R" Refactor recommendations
131
+ no-self-use,
132
+ no-else-return,
133
+ too-many-locals,
134
+ too-many-public-methods,
135
+ consider-using-set-comprehension,
136
+ inconsistent-return-statements,
137
+ useless-object-inheritance,
138
+ too-few-public-methods,
139
+ too-many-boolean-expressions,
140
+ too-many-instance-attributes,
141
+ too-many-return-statements,
142
+ literal-comparison,
143
+ too-many-statements,
144
+ too-many-ancestors,
145
+ literal-comparison,
146
+ consider-merging-isinstance,
147
+ too-many-nested-blocks,
148
+ trailing-comma-tuple,
149
+ simplifiable-if-statement,
150
+ consider-using-in,
151
+ consider-using-ternary,
152
+ too-many-arguments
153
+
154
+[REPORTS]
155
+# Tells whether to display a full report or only the messages.
156
+reports=no
157
+
158
+[BASIC]
159
+# Variable names can be 1 to 31 characters long, with lowercase and underscores
160
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
161
+
162
+# Argument names can be 2 to 31 characters long, with lowercase and underscores
163
+argument-rgx=[a-z_][a-z0-9_]{1,30}$
164
+
165
+# Method names should be at least 3 characters long
166
+# and be lowercased with underscores
167
+method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
168
+
169
+# Module names matching neutron-* are ok (files in bin/)
170
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|(neutron-[a-z0-9_-]+))$
171
+
172
+# Don't require docstrings on tests.
173
+no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$
174
+
175
+
176
+[FORMAT]
177
+# Maximum number of characters on a single line.
178
+max-line-length=79
179
+
180
+
181
+[VARIABLES]
182
+# List of additional names supposed to be defined in builtins. Remember that
183
+# you should avoid to define new builtins when possible.
184
+additional-builtins=_
185
+
186
+[TYPECHECK]
187
+# List of module names for which member attributes should not be checked
188
+ignored-modules=six.moves,_MovedItems
189
+

+ 56
- 0
tools/coding-checks.sh View File

@@ -0,0 +1,56 @@
1
+#!/bin/sh
2
+
3
+set -eu
4
+
5
+usage() {
6
+    echo "Usage: $0 [OPTION]..."
7
+    echo "Run Cinder's coding check(s)"
8
+    echo ""
9
+    echo " -Y, --pylint [<basecommit>] Run pylint check on the entire manila module or just files changed in basecommit (e.g. HEAD~1)"
10
+    echo " -h, --help   Print this usage message"
11
+    echo
12
+    exit 0
13
+}
14
+
15
+process_options() {
16
+    i=1
17
+    while [ $i -le $# ]; do
18
+        eval opt=\$$i
19
+        case $opt in
20
+            -h|--help) usage;;
21
+            -Y|--pylint) pylint=1;;
22
+            *) scriptargs="$scriptargs $opt"
23
+        esac
24
+        i=$((i+1))
25
+    done
26
+}
27
+
28
+run_pylint() {
29
+    local target="${scriptargs:-HEAD~1}"
30
+
31
+    if [[ "$target" = *"all"* ]]; then
32
+        files="manila"
33
+    else
34
+        files=$(git diff --name-only --diff-filter=ACMRU $target "*.py")
35
+    fi
36
+
37
+    if [ -n "${files}" ]; then
38
+        echo "Running pylint against:"
39
+        printf "\t%s\n" "${files[@]}"
40
+        pylint --rcfile=.pylintrc --output-format=colorized ${files} -E \
41
+            -j `python -c 'import multiprocessing as mp; print(mp.cpu_count())'`
42
+    else
43
+        echo "No python changes in this commit, pylint check not required."
44
+        exit 0
45
+    fi
46
+}
47
+
48
+scriptargs=
49
+pylint=1
50
+
51
+process_options $@
52
+
53
+if [ $pylint -eq 1 ]; then
54
+    run_pylint
55
+    exit 0
56
+fi

+ 0
- 232
tools/lintstack.py View File

@@ -1,232 +0,0 @@
1
-#!/usr/bin/env python
2
-
3
-# Copyright (c) 2013, AT&T Labs, Yun Mao <yunmao@gmail.com>
4
-# All Rights Reserved.
5
-#
6
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
7
-#    not use this file except in compliance with the License. You may obtain
8
-#    a copy of the License at
9
-#
10
-#         http://www.apache.org/licenses/LICENSE-2.0
11
-#
12
-#    Unless required by applicable law or agreed to in writing, software
13
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
-#    License for the specific language governing permissions and limitations
16
-#    under the License.
17
-
18
-"""pylint error checking."""
19
-
20
-import json
21
-import re
22
-import sys
23
-
24
-import six
25
-
26
-from pylint import lint
27
-from pylint.reporters import text
28
-
29
-# Note(maoy): E1103 is error code related to partial type inference
30
-ignore_codes = ["E1103"]
31
-# Note(maoy): the error message is the pattern of E0202. It should be ignored
32
-# for manila.tests modules
33
-# Note(chen): the second error message is the pattern of [E0611]
34
-# It should be ignored because use six module to keep py3.X compatibility.
35
-ignore_messages = ["An attribute affected in manila.tests",
36
-                   "No name 'urllib' in module '_MovedItems'"]
37
-# Note(maoy): we ignore all errors in openstack.common because it should be
38
-# checked elsewhere. We also ignore manila.tests for now due to high false
39
-# positive rate.
40
-ignore_modules = ["manila/openstack/common/", "manila/tests/"]
41
-
42
-KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"
43
-
44
-
45
-class LintOutput(object):
46
-
47
-    _cached_filename = None
48
-    _cached_content = None
49
-
50
-    def __init__(self, filename, lineno, line_content, code, message,
51
-                 lintoutput):
52
-        self.filename = filename
53
-        self.lineno = lineno
54
-        self.line_content = line_content
55
-        self.code = code
56
-        self.message = message
57
-        self.lintoutput = lintoutput
58
-
59
-    @classmethod
60
-    def from_line(cls, line):
61
-        m = re.search(r"(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line)
62
-        if m is None:
63
-            return None
64
-        matched = m.groups()
65
-        filename, lineno, code, message = (matched[0], int(matched[1]),
66
-                                           matched[2], matched[-1])
67
-        if cls._cached_filename != filename:
68
-            with open(filename) as f:
69
-                cls._cached_content = list(f.readlines())
70
-                cls._cached_filename = filename
71
-        line_content = cls._cached_content[lineno - 1].rstrip()
72
-        return cls(filename, lineno, line_content, code, message,
73
-                   line.rstrip())
74
-
75
-    @classmethod
76
-    def from_msg_to_dict(cls, msg):
77
-        """From the output of pylint msg, to a dict.
78
-
79
-        Each key is a unique error identifier, value is a list of
80
-        LintOutput.
81
-        """
82
-        result = {}
83
-        for line in msg.splitlines():
84
-            obj = cls.from_line(line)
85
-            if obj is None or obj.is_ignored():
86
-                continue
87
-            key = obj.key()
88
-            if key not in result:
89
-                result[key] = []
90
-            result[key].append(obj)
91
-        return result
92
-
93
-    def is_ignored(self):
94
-        if self.code in ignore_codes:
95
-            return True
96
-        if any(self.filename.startswith(name) for name in ignore_modules):
97
-            return True
98
-        if any(msg in self.message for msg in ignore_messages):
99
-            return True
100
-        return False
101
-
102
-    def key(self):
103
-        if self.code in ["E1101", "E1103"]:
104
-            # These two types of errors are like Foo class has no member bar.
105
-            # We discard the source code so that the error will be ignored
106
-            # next time another Foo.bar is encountered.
107
-            return self.message, ""
108
-        return self.message, self.line_content.strip()
109
-
110
-    def json(self):
111
-        return json.dumps(self.__dict__)
112
-
113
-    def review_str(self):
114
-        return ("File %(filename)s\nLine %(lineno)d:"
115
-                "%(line_content)s\n%(code)s: %(message)s" %
116
-                {'filename': self.filename, 'lineno': self.lineno,
117
-                 'line_content': self.line_content, 'code': self.code,
118
-                 'message': self.message})
119
-
120
-
121
-class ErrorKeys(object):
122
-
123
-    @classmethod
124
-    def print_json(cls, errors, output=sys.stdout):
125
-        print("# automatically generated by tools/lintstack.py", file=output)
126
-        for i in sorted(errors.keys()):
127
-            print(json.dumps(i), file=output)
128
-
129
-    @classmethod
130
-    def from_file(cls, filename):
131
-        keys = set()
132
-        for line in open(filename):
133
-            if line and line[0] != "#":
134
-                d = json.loads(line)
135
-                keys.add(tuple(d))
136
-        return keys
137
-
138
-
139
-def run_pylint():
140
-    buff = six.StringIO()
141
-    reporter = text.TextReporter(output=buff)
142
-    args = [
143
-        "--msg-template='{path}:{line}: [{msg_id}i({symbol}), {obj}] {msg}'",
144
-        "-E", "manila"]
145
-    lint.Run(args, reporter=reporter, exit=False)
146
-    val = buff.getvalue()
147
-    buff.close()
148
-    return val
149
-
150
-
151
-def generate_error_keys(msg=None):
152
-    print("Generating", KNOWN_PYLINT_EXCEPTIONS_FILE)
153
-    if msg is None:
154
-        msg = run_pylint()
155
-    errors = LintOutput.from_msg_to_dict(msg)
156
-    with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
157
-        ErrorKeys.print_json(errors, output=f)
158
-
159
-
160
-def check():
161
-    print("Running pylint. Be patient...")
162
-    newmsg = run_pylint()
163
-    errors = LintOutput.from_msg_to_dict(newmsg)
164
-
165
-    passed = True
166
-    for err_key, err_list in errors.items():
167
-        for err in err_list:
168
-            print(err.review_str() + "\n")
169
-            passed = False
170
-
171
-    if passed:
172
-        print("Congrats! pylint check passed.")
173
-    else:
174
-        print("\nPlease fix the errors above. If you believe they are false "
175
-              "positives, run 'tools/lintstack.py generate' to overwrite.")
176
-        sys.exit(1)
177
-
178
-
179
-def validate(newmsg=None):
180
-    print("Loading", KNOWN_PYLINT_EXCEPTIONS_FILE)
181
-    known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
182
-    if newmsg is None:
183
-        print("Running pylint. Be patient...")
184
-        newmsg = run_pylint()
185
-    errors = LintOutput.from_msg_to_dict(newmsg)
186
-
187
-    print("Unique errors reported by pylint: was %d, now %d."
188
-          % (len(known), len(errors)))
189
-    passed = True
190
-    for err_key, err_list in errors.items():
191
-        for err in err_list:
192
-            if err_key not in known:
193
-                print(err.lintoutput)
194
-                print()
195
-                passed = False
196
-    if passed:
197
-        print("Congrats! pylint check passed.")
198
-        redundant = known - set(errors.keys())
199
-        if redundant:
200
-            print("Extra credit: some known pylint exceptions disappeared.")
201
-            for i in sorted(redundant):
202
-                print(json.dumps(i))
203
-            print("Consider regenerating the exception file if you will.")
204
-    else:
205
-        print("Please fix the errors above. If you believe they are false "
206
-              "positives, run 'tools/lintstack.py generate' to overwrite.")
207
-        sys.exit(1)
208
-
209
-
210
-def usage():
211
-    print("""Usage: tools/lintstack.py [generate|validate]
212
-    To generate pylint_exceptions file: tools/lintstack.py generate
213
-    To validate the current commit: tools/lintstack.py
214
-    """)
215
-
216
-
217
-def main():
218
-    option = "validate"
219
-    if len(sys.argv) > 1:
220
-        option = sys.argv[1]
221
-    if option == "generate":
222
-        generate_error_keys()
223
-    elif option == "validate":
224
-        validate()
225
-    elif option == "check":
226
-        check()
227
-    else:
228
-        usage()
229
-
230
-
231
-if __name__ == "__main__":
232
-    main()

+ 0
- 59
tools/lintstack.sh View File

@@ -1,59 +0,0 @@
1
-#!/usr/bin/env bash
2
-
3
-# Copyright (c) 2012-2013, AT&T Labs, Yun Mao <yunmao@gmail.com>
4
-# All Rights Reserved.
5
-#
6
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
7
-#    not use this file except in compliance with the License. You may obtain
8
-#    a copy of the License at
9
-#
10
-#         http://www.apache.org/licenses/LICENSE-2.0
11
-#
12
-#    Unless required by applicable law or agreed to in writing, software
13
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
-#    License for the specific language governing permissions and limitations
16
-#    under the License.
17
-
18
-# Use lintstack.py to compare pylint errors.
19
-# We run pylint twice, once on HEAD, once on the code before the latest
20
-# commit for review.
21
-set -e
22
-TOOLS_DIR=$(cd $(dirname "$0") && pwd)
23
-# Get the current branch name.
24
-GITHEAD=`git rev-parse --abbrev-ref HEAD`
25
-if [[ "$GITHEAD" == "HEAD" ]]; then
26
-    # In detached head mode, get revision number instead
27
-    GITHEAD=`git rev-parse HEAD`
28
-    echo "Currently we are at commit $GITHEAD"
29
-else
30
-    echo "Currently we are at branch $GITHEAD"
31
-fi
32
-
33
-cp -f $TOOLS_DIR/lintstack.py $TOOLS_DIR/lintstack.head.py
34
-
35
-if git rev-parse HEAD^2 2>/dev/null; then
36
-    # The HEAD is a Merge commit. Here, the patch to review is
37
-    # HEAD^2, the master branch is at HEAD^1, and the patch was
38
-    # written based on HEAD^2~1.
39
-    PREV_COMMIT=`git rev-parse HEAD^2~1`
40
-    git checkout HEAD~1
41
-    # The git merge is necessary for reviews with a series of patches.
42
-    # If not, this is a no-op so won't hurt either.
43
-    git merge $PREV_COMMIT
44
-else
45
-    # The HEAD is not a merge commit. This won't happen on gerrit.
46
-    # Most likely you are running against your own patch locally.
47
-    # We assume the patch to examine is HEAD, and we compare it against
48
-    # HEAD~1
49
-    git checkout HEAD~1
50
-fi
51
-
52
-# First generate tools/pylint_exceptions from HEAD~1
53
-$TOOLS_DIR/lintstack.head.py generate
54
-# Then use that as a reference to compare against HEAD
55
-git checkout $GITHEAD
56
-$TOOLS_DIR/lintstack.head.py
57
-echo "Check passed. FYI: the pylint exceptions are:"
58
-cat $TOOLS_DIR/pylint_exceptions
59
-

+ 2
- 2
tox.ini View File

@@ -104,9 +104,9 @@ commands =
104 104
 [testenv:pylint]
105 105
 basepython = python3
106 106
 deps = -r{toxinidir}/requirements.txt
107
-       pylint==1.9.0
107
+       pylint==2.1.1
108 108
 whitelist_externals = bash
109
-commands = bash tools/lintstack.sh
109
+commands = bash ./tools/coding-checks.sh --pylint {posargs}
110 110
 
111 111
 [testenv:lint]
112 112
 basepython = python3

Loading…
Cancel
Save