murano/tools/lintstack.py

#!/usr/bin/env python
# Copyright (c) 2015 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""pylint error checking."""

import io
import json
import os
import re
import sys

from pylint import lint
from pylint.reporters import text

# enabled checks
# http://pylint-messages.wikidot.com/all-codes
ENABLED_CODES = (
    # refactor category
    "R0801", "R0911", "R0912", "R0913", "R0914", "R0915",
    # warning category
    "W0612", "W0613", "W0703",
    # convention category
    "C1001")

LINE_PATTERN = r"(\S+):(\d+): \[(\S+)(, \S*)?] (.*)"

KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"


class LintOutput(object):

    _cached_filename = None
    _cached_content = None

    def __init__(self, filename, lineno, line_content, code, message,
                 lintoutput, additional_content):
        self.filename = filename
        self.lineno = lineno
        self.line_content = line_content
        self.code = code
        self.message = message
        self.lintoutput = lintoutput
        self.additional_content = additional_content

    @classmethod
    def get_duplicate_code_location(cls, remaining_lines):
        module, lineno = remaining_lines.pop(0)[2:].split(":")
        filename = module.replace(".", os.sep) + ".py"
        return filename, int(lineno)

    @classmethod
    def get_line_content(cls, filename, lineno):
        if cls._cached_filename != filename:
            with open(filename) as f:
                cls._cached_content = list(f.readlines())
                cls._cached_filename = filename
        # find first non-empty line
        lineno -= 1
        while True:
            line_content = cls._cached_content[lineno].rstrip()
            lineno += 1
            if line_content:
                return line_content

    @classmethod
    def from_line(cls, line, remaining_lines):
        m = re.search(LINE_PATTERN, line)
        if not m:
            return None
        matched = m.groups()
        filename, lineno, code, message = (matched[0], int(matched[1]),
                                           matched[2], matched[-1])
        additional_content = None

        # duplicate code output needs special handling
        if "duplicate-code" in code:
            line_count = 0
            for next_line in remaining_lines:
                if re.search(LINE_PATTERN, next_line):
                    break
                line_count += 1
            if line_count:
                additional_content = remaining_lines[0:line_count]

            filename, lineno = cls.get_duplicate_code_location(remaining_lines)
            # fixes incorrectly reported file path
            line = line.replace(matched[0], filename)
            line = line.replace(":%s:" % matched[1], "")

        line_content = cls.get_line_content(filename, lineno)
        return cls(filename, lineno, line_content, code, message,
                   line.rstrip(), additional_content)

    @classmethod
    def from_msg_to_dict(cls, msg):
        """Creates dict from pylint msg

        From the output of pylint msg, to a dict, where each key
        is a unique error identifier, value is a list of LintOutput
        """
        result = {}
        lines = msg.splitlines()
        while lines:
            line = lines.pop(0)
            obj = cls.from_line(line, lines)
            if not obj:
                continue
            key = obj.key()
            if key not in result:
                result[key] = []
            result[key].append(obj)
        return result

    def key(self):
        return self.message, self.line_content.strip()

    def json(self):
        return json.dumps(self.__dict__)

    def review_str(self):
        kargs = {"filename": self.filename,
                 "lineno": self.lineno,
                 "line_content": self.line_content,
                 "code": self.code,
                 "message": self.message}
        return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n"
                "%(code)s: %(message)s" % kargs)


class ErrorKeys(object):

    @classmethod
    def print_json(cls, errors, output=sys.stdout):
        print("# automatically generated by tools/lintstack.py", file=output)
        for i in sorted(errors.keys()):
            print(json.dumps(i), file=output)

    @classmethod
    def from_file(cls, filename):
        keys = set()
        for line in open(filename):
            if line and line[0] != "#":
                d = json.loads(line)
                keys.add(tuple(d))
        return keys


def run_pylint():
    buff = io.StringIO()
    reporter = text.ParseableTextReporter(output=buff)
    args = ["-rn", "--disable=all", "--enable=" + ",".join(ENABLED_CODES),
            "murano"]
    lint.Run(args, reporter=reporter, exit=False)
    val = buff.getvalue()
    buff.close()
    return val


def generate_error_keys(msg=None):
    print("Generating", KNOWN_PYLINT_EXCEPTIONS_FILE)
    if msg is None:
        msg = run_pylint()

    errors = LintOutput.from_msg_to_dict(msg)
    with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
        ErrorKeys.print_json(errors, output=f)


def validate(newmsg=None):
    print("Loading", KNOWN_PYLINT_EXCEPTIONS_FILE)
    known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
    if newmsg is None:
        print("Running pylint. Be patient...")
        newmsg = run_pylint()
    errors = LintOutput.from_msg_to_dict(newmsg)

    print()
    print("Unique errors reported by pylint: was %d, now %d."
          % (len(known), len(errors)))
    passed = True
    for err_key, err_list in errors.items():
        for err in err_list:
            if err_key not in known:
                print()
                print(err.lintoutput)
                print(err.review_str())
                if err.additional_content:
                    max_len = max(map(len, err.additional_content))
                    print("-" * max_len)
                    print(os.linesep.join(err.additional_content))
                    print("-" * max_len)
                passed = False
    if passed:
        print("Congrats! pylint check passed.")
        redundant = known - set(errors.keys())
        if redundant:
            print("Extra credit: some known pylint exceptions disappeared.")
            for i in sorted(redundant):
                print(json.dumps(i))
            print("Consider regenerating the exception file if you will.")
    else:
        print()
        print("Please fix the errors above. If you believe they are false "
              "positives, run 'tools/lintstack.py generate' to overwrite.")
        sys.exit(1)


def usage():
    print("""Usage: tools/lintstack.py [generate|validate]
    To generate pylint_exceptions file: tools/lintstack.py generate
    To validate the current commit: tools/lintstack.py
    """)


def main():
    option = "validate"
    if len(sys.argv) > 1:
        option = sys.argv[1]
    if option == "generate":
        generate_error_keys()
    elif option == "validate":
        validate()
    else:
        usage()


if __name__ == "__main__":
    main()