diff --git a/.gitignore b/.gitignore index bfdf4faab..e62855513 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ include/* local/* src/* build/* +.ve diff --git a/Makefile b/Makefile index 7f5c906e7..d5f0314f8 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,30 @@ -default: - pyflakes teeth_agent/ - pep8 --max-line-length=119 teeth_agent/ +CODEDIR=teeth_agent +SCRIPTSDIR=scripts + +UNITTESTS ?= ${CODEDIR} +PYTHONLINT=${SCRIPTSDIR}/python-lint.py +PYDIRS=${CODEDIR} ${SCRIPTSDIR} + +test: unit + +unit: +ifneq ($(JENKINS_URL), ) + trial --random 0 --reporter=subunit ${UNITTESTS} | tee subunit-output.txt + tail -n +3 subunit-output.txt | subunit2junitxml > test-report.xml +else + trial --random 0 ${UNITTESTS} +endif + +env: + ./scripts/bootstrap-virtualenv.sh + +lint: + ${PYTHONLINT} ${PYDIRS} + +clean: + find . -name '*.pyc' -delete + find . -name '.coverage' -delete + find . -name '_trial_coverage' -print0 | xargs rm -rf + find . -name '_trial_temp' -print0 | xargs rm -rf + rm -rf dist build *.egg-info + diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 000000000..4203a6ded --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +pep257==0.2.4 +plumbum==1.3.0 +pep8==1.4.6 +pyflakes==0.7.3 diff --git a/requirements.txt b/requirements.txt index cf1c8bff7..79d73fe64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ Twisted==13.1.0 argparse==1.2.1 distribute==0.6.24 -pep8==1.4.6 -pyflakes==0.7.3 simplejson==3.3.0 wsgiref==0.1.2 zope.interface==4.0.5 diff --git a/scripts/bootstrap-virtualenv.sh b/scripts/bootstrap-virtualenv.sh new file mode 100755 index 000000000..751941672 --- /dev/null +++ b/scripts/bootstrap-virtualenv.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Create an initial virtualenv based on the VE_DIR environment variable (.ve) +# by default. This is used by the Makefile `make env` to allow bootstrapping in +# environments where virtualenvwrapper is unavailable or unappropriate. Such +# as on Jenkins. +# + +set -e + +VE_DIR=${VE_DIR:=.ve} + +if [[ -d ${VE_DIR} ]]; then + echo "Skipping build virtualenv" +else + echo "Building complete virtualenv" + virtualenv ${VE_DIR} +fi + +source ${VE_DIR}/bin/activate + +pip install -r requirements.txt -r dev-requirements.txt + diff --git a/scripts/python-lint.py b/scripts/python-lint.py new file mode 100755 index 000000000..645f8eb6a --- /dev/null +++ b/scripts/python-lint.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +""" +Enforces Python coding standards via pep8, pyflakes and pylint + + +Installation: +pip install pep8 - style guide +pip install pep257 - for docstrings +pip install pyflakes - unused imports and variable declarations +pip install plumbum - used for executing shell commands + +This script can be called from the git pre-commit hook with a +--git-precommit option +""" + +import os +import pep257 +import re +import sys +from plumbum import local, cli, commands + +pep8_options = [ + '--max-line-length=105' +] + + +def lint(to_lint): + """ + Run all linters against a list of files. + + :param to_lint: a list of files to lint. + + """ + exit_code = 0 + for linter, options in (('pyflakes', []), ('pep8', pep8_options)): + try: + output = local[linter](*(options + to_lint)) + except commands.ProcessExecutionError as e: + output = e.stdout + + if output: + exit_code = 1 + print "{0} Errors:".format(linter) + print output + + output = hacked_pep257(to_lint) + if output: + exit_code = 1 + print "Docstring Errors:".format(linter.upper()) + print output + + sys.exit(exit_code) + + +def hacked_pep257(to_lint): + """ + Check for the presence of docstrings, but ignore some of the options + """ + def ignore(*args, **kwargs): + pass + + pep257.check_blank_before_after_class = ignore + pep257.check_blank_after_last_paragraph = ignore + pep257.check_blank_after_summary = ignore + pep257.check_ends_with_period = ignore + pep257.check_one_liners = ignore + pep257.check_imperative_mood = ignore + + original_check_return_type = pep257.check_return_type + + def better_check_return_type(def_docstring, context, is_script): + """ + Ignore private methods + """ + def_name = context.split()[1] + if def_name.startswith('_') and not def_name.endswith('__'): + original_check_return_type(def_docstring, context, is_script) + + pep257.check_return_type = better_check_return_type + + errors = [] + for filename in to_lint: + with open(filename) as f: + source = f.read() + if source: + errors.extend(pep257.check_source(source, filename)) + return '\n'.join([str(error) for error in sorted(errors)]) + + +class Lint(cli.Application): + """ + Command line app for VmrunWrapper + """ + + DESCRIPTION = "Lints python with pep8, pep257, and pyflakes" + + git = cli.Flag("--git-precommit", help="Lint only modified git files", + default=False) + + def main(self, *directories): + """ + The actual logic that runs the linters + """ + if not self.git and len(directories) == 0: + print ("ERROR: At least one directory must be provided (or the " + "--git-precommit flag must be passed.\n") + self.help() + return + + if len(directories) > 0: + find = local['find'] + files = [] + for directory in directories: + real = os.path.expanduser(directory) + if not os.path.exists(real): + raise ValueError("{0} does not exist".format(directory)) + files.extend(find(real, '-not', '-name', '._*', '-name', '*.py').strip().split('\n')) + else: + status = local['git']('status', '--porcelain', '-uno') + root = local['git']('rev-parse', '--show-toplevel').strip() + + # get all modified or added python files + modified = re.findall(r"^\s[AM]\s+(\S+\.py)$", status, re.MULTILINE) + + # now just get the path part, which all should be relative to the + # root + files = [os.path.join(root, line.split(' ', 1)[-1].strip()) + for line in modified] + + if len(files) > 0: + print "Linting {0} python files.\n".format(len(files)) + lint(files) + else: + print "No python files found to lint.\n" + + +if __name__ == "__main__": + Lint.run()