diff --git a/savanna/service/api.py b/savanna/service/api.py index eaa0c08a..7500b457 100644 --- a/savanna/service/api.py +++ b/savanna/service/api.py @@ -16,10 +16,10 @@ import eventlet from oslo.config import cfg +from savanna import exceptions as ex from savanna.openstack.common import log as logging from savanna.service import cluster_ops import savanna.storage.storage as storage -from savanna import exceptions as ex LOG = logging.getLogger(__name__) diff --git a/savanna/service/cluster_ops.py b/savanna/service/cluster_ops.py index a201c7af..46a6eb84 100644 --- a/savanna/service/cluster_ops.py +++ b/savanna/service/cluster_ops.py @@ -119,7 +119,7 @@ def launch_cluster(headers, cluster): configs = dict() for cf in nc.node_template.node_template_configs: proc_name = cf.node_process_property.node_process.name - if not proc_name in configs: + if proc_name not in configs: configs[proc_name] = dict() name = cf.node_process_property.name @@ -276,7 +276,8 @@ def _keys_exist(map, key1, key2): def _extract_environment_confs(node_configs): - "Returns list of Hadoop parameters which should be passed via environment" + """Returns list of Hadoop parameters which should be passed via environment + """ lst = [] @@ -292,7 +293,8 @@ def _extract_environment_confs(node_configs): def _extract_xml_confs(node_configs): """Returns list of Hadoop parameters which should be passed into general - configs like core-site.xml""" + configs like core-site.xml + """ # For now we assume that all parameters outside of ENV_CONFS # are passed to xml files diff --git a/tools/build_docs b/tools/build_docs index a46a5ccc..6918edf3 100755 --- a/tools/build_docs +++ b/tools/build_docs @@ -1,19 +1,3 @@ #!/bin/bash -e -if [ ${VIRTUAL_ENV:+x} ] - then echo "VIRTUAL_ENV is already specified" - else export VIRTUAL_ENV=.venv -fi - -echo "VIRTUAL_ENV = $VIRTUAL_ENV" - -echo "Building docs..." -echo "================" - -rm -rf doc/html doc/build -rm -rf doc/source/apidoc doc/source/api - -$VIRTUAL_ENV/bin/python setup.py build_sphinx - -echo "Done." -echo "Docs are now available in 'doc/build/html/'" +tox -edocs diff --git a/tools/get_auth_token b/tools/get_auth_token index 42b91350..d810ea50 100755 --- a/tools/get_auth_token +++ b/tools/get_auth_token @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/../.venv/bin/python $(dirname $0)/get_auth_token.py $* +tox -evenv -- python $(dirname $0)/get_auth_token.py $* diff --git a/tools/hacking.py b/tools/hacking.py deleted file mode 100644 index 04307eef..00000000 --- a/tools/hacking.py +++ /dev/null @@ -1,668 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012, Cloudscaling -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""nova HACKING file compliance testing - -Built on top of pep8.py -""" - -import imp -import inspect -import logging -import os -import re -import subprocess -import sys -import tokenize -import traceback - -import pep8 - -# Don't need this for testing -logging.disable('LOG') - -#N1xx comments -#N2xx except -#N3xx imports -#N4xx docstrings -#N5xx dictionaries/lists -#N6xx calling methods -#N7xx localization -#N8xx git commit messages -#N9xx other - -IMPORT_EXCEPTIONS = ['sqlalchemy', 'migrate', 'nova.db.sqlalchemy.session', - 'nova.db.sqlalchemy.migration.versioning_api'] -# Paste is missing a __init__ in top level directory -START_DOCSTRING_TRIPLE = ['u"""', 'r"""', '"""', "u'''", "r'''", "'''"] -END_DOCSTRING_TRIPLE = ['"""', "'''"] -VERBOSE_MISSING_IMPORT = os.getenv('HACKING_VERBOSE_MISSING_IMPORT', 'False') - -_missingImport = set([]) - - -# Monkey patch broken excluded filter in pep8 -# See https://github.com/jcrocholl/pep8/pull/111 -def excluded(self, filename): - """Check if options.exclude contains a pattern that matches filename.""" - basename = os.path.basename(filename) - return any((pep8.filename_match(filename, self.options.exclude, - default=False), - pep8.filename_match(basename, self.options.exclude, - default=False))) - - -def input_dir(self, dirname): - """Check all files in this directory and all subdirectories.""" - dirname = dirname.rstrip('/') - if self.excluded(dirname): - return 0 - counters = self.options.report.counters - verbose = self.options.verbose - filepatterns = self.options.filename - runner = self.runner - for root, dirs, files in os.walk(dirname): - if verbose: - print('directory ' + root) - counters['directories'] += 1 - for subdir in sorted(dirs): - if self.excluded(os.path.join(root, subdir)): - dirs.remove(subdir) - for filename in sorted(files): - # contain a pattern that matches? - if ((pep8.filename_match(filename, filepatterns) and - not self.excluded(filename))): - runner(os.path.join(root, filename)) - - -def is_import_exception(mod): - return (mod in IMPORT_EXCEPTIONS or - any(mod.startswith(m + '.') for m in IMPORT_EXCEPTIONS)) - - -def import_normalize(line): - # convert "from x import y" to "import x.y" - # handle "from x import y as z" to "import x.y as z" - split_line = line.split() - if ("import" in line and line.startswith("from ") and "," not in line and - split_line[2] == "import" and split_line[3] != "*" and - split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): - return "import %s.%s" % (split_line[1], split_line[3]) - else: - return line - - -def nova_todo_format(physical_line, tokens): - """Check for 'TODO()'. - - nova HACKING guide recommendation for TODO: - Include your name with TODOs as in "#TODO(termie)" - - Okay: #TODO(sdague) - N101: #TODO fail - N101: #TODO (jogo) fail - """ - # TODO(sdague): TODO check shouldn't fail inside of space - pos = physical_line.find('TODO') - pos1 = physical_line.find('TODO(') - pos2 = physical_line.find('#') # make sure it's a comment - if (pos != pos1 and pos2 >= 0 and pos2 < pos and len(tokens) == 0): - return pos, "N101: Use TODO(NAME)" - - -def nova_except_format(logical_line): - r"""Check for 'except:'. - - nova HACKING guide recommends not using except: - Do not write "except:", use "except Exception:" at the very least - - Okay: except Exception: - N201: except: - """ - if logical_line.startswith("except:"): - yield 6, "N201: no 'except:' at least use 'except Exception:'" - - -def nova_except_format_assert(logical_line): - r"""Check for 'assertRaises(Exception'. - - nova HACKING guide recommends not using assertRaises(Exception...): - Do not use overly broad Exception type - - Okay: self.assertRaises(NovaException) - N202: self.assertRaises(Exception) - """ - if logical_line.startswith("self.assertRaises(Exception"): - yield 1, "N202: assertRaises Exception too broad" - - -modules_cache = dict((mod, True) for mod in tuple(sys.modules.keys()) - + sys.builtin_module_names) - -RE_RELATIVE_IMPORT = re.compile('^from\s*[.]') - - -def nova_import_rules(logical_line): - r"""Check for imports. - - nova HACKING guide recommends one import per line: - Do not import more than one module per line - - Examples: - Okay: from nova.compute import api - N301: from nova.compute import api, utils - - - Imports should usually be on separate lines. - - nova HACKING guide recommends importing only modules: - Do not import objects, only modules - - Examples: - Okay: from os import path - Okay: from os import path as p - Okay: from os import (path as p) - Okay: import os.path - Okay: from nova.compute import rpcapi - N302: from os.path import dirname as dirname2 - N302: from os.path import (dirname as dirname2) - N303: from os.path import * - N304: from .compute import rpcapi - """ - #NOTE(afazekas): An old style relative import example will not be able to - # pass the doctest, since the relativity depends on the file's locality - - def is_module_for_sure(mod, search_path=sys.path): - mod = mod.replace('(', '') # Ignore parentheses - try: - mod_name = mod - while '.' in mod_name: - pack_name, _sep, mod_name = mod.partition('.') - f, p, d = imp.find_module(pack_name, search_path) - search_path = [p] - imp.find_module(mod_name, search_path) - except ImportError: - try: - # NOTE(vish): handle namespace modules - module = __import__(mod) - except ImportError, exc: - # NOTE(vish): the import error might be due - # to a missing dependency - missing = str(exc).split()[-1] - if (missing != mod.split('.')[-1] or - "cannot import" in str(exc)): - _missingImport.add(missing) - return True - return False - except Exception, exc: - # NOTE(jogo) don't stack trace if unexpected import error, - # log and continue. - traceback.print_exc() - return False - return True - - def is_module(mod): - """Checks for non module imports.""" - if mod in modules_cache: - return modules_cache[mod] - res = is_module_for_sure(mod) - modules_cache[mod] = res - return res - - current_path = os.path.dirname(pep8.current_file) - current_mod = os.path.basename(pep8.current_file) - if current_mod[-3:] == ".py": - current_mod = current_mod[:-3] - - split_line = logical_line.split() - split_line_len = len(split_line) - if (split_line[0] in ('import', 'from') and split_line_len > 1 and - not is_import_exception(split_line[1])): - pos = logical_line.find(',') - if pos != -1: - if split_line[0] == 'from': - yield pos, "N301: one import per line" - return # ',' is not supported by the N302 checker yet - pos = logical_line.find('*') - if pos != -1: - yield pos, "N303: No wildcard (*) import." - return - - if split_line_len in (2, 4, 6) and split_line[1] != "__future__": - if 'from' == split_line[0] and split_line_len > 3: - mod = '.'.join((split_line[1], split_line[3])) - if is_import_exception(mod): - return - if RE_RELATIVE_IMPORT.search(logical_line): - yield logical_line.find('.'), ("N304: No " - "relative imports. '%s' is a relative import" - % logical_line) - return - - if not is_module(mod): - yield 0, ("N302: import only modules." - "'%s' does not import a module" % logical_line) - return - - #NOTE(afazekas): import searches first in the package - # The import keyword just imports modules - # The guestfs module now imports guestfs - mod = split_line[1] - if (current_mod != mod and - not is_module(mod) and - is_module_for_sure(mod, [current_path])): - yield 0, ("N304: No relative imports." - " '%s' is a relative import" - % logical_line) - - -#TODO(jogo): import template: N305 - - -def nova_import_alphabetical(logical_line, blank_lines, previous_logical, - indent_level, previous_indent_level): - r"""Check for imports in alphabetical order. - - nova HACKING guide recommendation for imports: - imports in human alphabetical order - - Okay: import os\nimport sys\n\nimport nova\nfrom nova import test - N306: import sys\nimport os - """ - # handle import x - # use .lower since capitalization shouldn't dictate order - split_line = import_normalize(logical_line.strip()).lower().split() - split_previous = import_normalize(previous_logical.strip()).lower().split() - - if blank_lines < 1 and indent_level == previous_indent_level: - length = [2, 4] - if (len(split_line) in length and len(split_previous) in length and - split_line[0] == "import" and split_previous[0] == "import"): - if split_line[1] < split_previous[1]: - yield (0, "N306: imports not in alphabetical order (%s, %s)" - % (split_previous[1], split_line[1])) - - -def nova_import_no_db_in_virt(logical_line, filename): - """Check for db calls from nova/virt - - As of grizzly-2 all the database calls have been removed from - nova/virt, and we want to keep it that way. - - N307 - """ - if "nova/virt" in filename and not filename.endswith("fake.py"): - if logical_line.startswith("from nova import db"): - yield (0, "N307: nova.db import not allowed in nova/virt/*") - - -def is_docstring(physical_line, previous_logical): - """Return True if found docstring - 'A docstring is a string literal that occurs as the first statement in a - module, function, class,' - http://www.python.org/dev/peps/pep-0257/#what-is-a-docstring - """ - line = physical_line.lstrip() - start = max([line.find(i) for i in START_DOCSTRING_TRIPLE]) - end = max([line[-4:-1] == i for i in END_DOCSTRING_TRIPLE]) - if (previous_logical.startswith("def ") or - previous_logical.startswith("class ")): - if start is 0: - return True - else: - # Handle multi line comments - return end and start in (-1, len(line) - 4) - - -def nova_docstring_start_space(physical_line, previous_logical): - r"""Check for docstring not start with space. - - nova HACKING guide recommendation for docstring: - Docstring should not start with space - - Okay: def foo():\n '''This is good.''' - Okay: def foo():\n a = ''' This is not a docstring.''' - Okay: def foo():\n pass\n ''' This is not.''' - N401: def foo():\n ''' This is not.''' - """ - # short circuit so that we don't fail on our own fail test - # when running under external pep8 - if physical_line.find("N401: def foo()") != -1: - return - - # it's important that we determine this is actually a docstring, - # and not a doc block used somewhere after the first line of a - # function def - if is_docstring(physical_line, previous_logical): - pos = max([physical_line.find(i) for i in START_DOCSTRING_TRIPLE]) - if physical_line[pos + 3] == ' ': - return (pos, "N401: docstring should not start with" - " a space") - - -def nova_docstring_one_line(physical_line, previous_logical): - r"""Check one line docstring end. - - nova HACKING guide recommendation for one line docstring: - A one line docstring looks like this and ends in punctuation. - - Okay: def foo():\n '''This is good.''' - Okay: def foo():\n '''This is good too!''' - Okay: def foo():\n '''How about this?''' - Okay: def foo():\n a = '''This is not a docstring''' - Okay: def foo():\n pass\n '''This is not a docstring''' - Okay: class Foo:\n pass\n '''This is not a docstring''' - N402: def foo():\n '''This is not''' - N402: def foo():\n '''Bad punctuation,''' - N402: class Foo:\n '''Bad punctuation,''' - """ - #TODO(jogo) make this apply to multi line docstrings as well - line = physical_line.lstrip() - if is_docstring(physical_line, previous_logical): - pos = max([line.find(i) for i in START_DOCSTRING_TRIPLE]) # start - end = max([line[-4:-1] == i for i in END_DOCSTRING_TRIPLE]) # end - - if pos != -1 and end and len(line) > pos + 4: - if line[-5] not in ['.', '?', '!']: - return pos, "N402: one line docstring needs punctuation." - - -def nova_docstring_multiline_end(physical_line, previous_logical, tokens): - r"""Check multi line docstring end. - - nova HACKING guide recommendation for docstring: - Docstring should end on a new line - - Okay: '''foobar\nfoo\nbar\n''' - Okay: def foo():\n '''foobar\nfoo\nbar\n''' - Okay: class Foo:\n '''foobar\nfoo\nbar\n''' - Okay: def foo():\n a = '''not\na\ndocstring''' - Okay: def foo():\n pass\n'''foobar\nfoo\nbar\n d''' - N403: def foo():\n '''foobar\nfoo\nbar\ndocstring''' - N403: class Foo:\n '''foobar\nfoo\nbar\ndocstring'''\n\n - """ - # if find OP tokens, not a docstring - ops = [t for t, _, _, _, _ in tokens if t == tokenize.OP] - if (is_docstring(physical_line, previous_logical) and len(tokens) > 0 and - len(ops) == 0): - pos = max(physical_line.find(i) for i in END_DOCSTRING_TRIPLE) - if physical_line.strip() not in START_DOCSTRING_TRIPLE: - return (pos, "N403: multi line docstring end on new line") - - -def nova_docstring_multiline_start(physical_line, previous_logical, tokens): - r"""Check multi line docstring start with summary. - - nova HACKING guide recommendation for docstring: - Docstring should start with A multi line docstring has a one-line summary - - Okay: '''foobar\nfoo\nbar\n''' - Okay: def foo():\n a = '''\nnot\na docstring\n''' - N404: def foo():\n'''\nfoo\nbar\n'''\n\n - """ - if is_docstring(physical_line, previous_logical): - pos = max([physical_line.find(i) for i in START_DOCSTRING_TRIPLE]) - # start of docstring when len(tokens)==0 - if len(tokens) == 0 and pos != -1 and len(physical_line) == pos + 4: - if physical_line.strip() in START_DOCSTRING_TRIPLE: - return (pos, "N404: multi line docstring " - "should start with a summary") - - -def nova_no_cr(physical_line): - r"""Check that we only use newlines not carriage returns. - - Okay: import os\nimport sys - # pep8 doesn't yet replace \r in strings, will work on an - # upstream fix - N901 import os\r\nimport sys - """ - pos = physical_line.find('\r') - if pos != -1 and pos == (len(physical_line) - 2): - return (pos, "N901: Windows style line endings not allowed in code") - - -FORMAT_RE = re.compile("%(?:" - "%|" # Ignore plain percents - "(\(\w+\))?" # mapping key - "([#0 +-]?" # flag - "(?:\d+|\*)?" # width - "(?:\.\d+)?" # precision - "[hlL]?" # length mod - "\w))") # type - - -class LocalizationError(Exception): - pass - - -def check_i18n(): - """Generator that checks token stream for localization errors. - - Expects tokens to be ``send``ed one by one. - Raises LocalizationError if some error is found. - """ - while True: - try: - token_type, text, _, _, line = yield - except GeneratorExit: - return - - if (token_type == tokenize.NAME and text == "_" and - not line.startswith('def _(msg):')): - - while True: - token_type, text, start, _, _ = yield - if token_type != tokenize.NL: - break - if token_type != tokenize.OP or text != "(": - continue # not a localization call - - format_string = '' - while True: - token_type, text, start, _, _ = yield - if token_type == tokenize.STRING: - format_string += eval(text) - elif token_type == tokenize.NL: - pass - else: - break - - if not format_string: - raise LocalizationError(start, - "N701: Empty localization string") - if token_type != tokenize.OP: - raise LocalizationError(start, - "N701: Invalid localization call") - if text != ")": - if text == "%": - raise LocalizationError(start, - "N702: Formatting operation should be outside" - " of localization method call") - elif text == "+": - raise LocalizationError(start, - "N702: Use bare string concatenation instead" - " of +") - else: - raise LocalizationError(start, - "N702: Argument to _ must be just a string") - - format_specs = FORMAT_RE.findall(format_string) - positional_specs = [(key, spec) for key, spec in format_specs - if not key and spec] - # not spec means %%, key means %(smth)s - if len(positional_specs) > 1: - raise LocalizationError(start, - "N703: Multiple positional placeholders") - - -def nova_localization_strings(logical_line, tokens): - r"""Check localization in line. - - Okay: _("This is fine") - Okay: _("This is also fine %s") - N701: _('') - N702: _("Bob" + " foo") - N702: _("Bob %s" % foo) - # N703 check is not quite right, disabled by removing colon - N703 _("%s %s" % (foo, bar)) - """ - # TODO(sdague) actually get these tests working - gen = check_i18n() - next(gen) - try: - map(gen.send, tokens) - gen.close() - except LocalizationError as e: - yield e.args - -#TODO(jogo) Dict and list objects - - -def nova_is_not(logical_line): - r"""Check localization in line. - - Okay: if x is not y - N901: if not X is Y - N901: if not X.B is Y - """ - split_line = logical_line.split() - if (len(split_line) == 5 and split_line[0] == 'if' and - split_line[1] == 'not' and split_line[3] == 'is'): - yield (logical_line.find('not'), "N901: Use the 'is not' " - "operator for when testing for unequal identities") - - -def nova_not_in(logical_line): - r"""Check localization in line. - - Okay: if x not in y - Okay: if not (X in Y or X is Z) - Okay: if not (X in Y) - N902: if not X in Y - N902: if not X.B in Y - """ - split_line = logical_line.split() - if (len(split_line) == 5 and split_line[0] == 'if' and - split_line[1] == 'not' and split_line[3] == 'in' and not - split_line[2].startswith('(')): - yield (logical_line.find('not'), "N902: Use the 'not in' " - "operator for collection membership evaluation") - -current_file = "" - - -def readlines(filename): - """Record the current file being tested.""" - pep8.current_file = filename - return open(filename).readlines() - - -def add_nova(): - """Monkey patch in nova guidelines. - - Look for functions that start with nova_ and have arguments - and add them to pep8 module - Assumes you know how to write pep8.py checks - """ - for name, function in globals().items(): - if not inspect.isfunction(function): - continue - args = inspect.getargspec(function)[0] - if args and name.startswith("nova"): - exec("pep8.%s = %s" % (name, name)) - - -def once_git_check_commit_title(): - """Check git commit messages. - - nova HACKING recommends not referencing a bug or blueprint in first line, - it should provide an accurate description of the change - N801 - N802 Title limited to 72 chars - """ - #Get title of most recent commit - - subp = subprocess.Popen(['git', 'log', '--no-merges', '--pretty=%s', '-1'], - stdout=subprocess.PIPE) - title = subp.communicate()[0] - if subp.returncode: - raise Exception("git log failed with code %s" % subp.returncode) - - #From https://github.com/openstack/openstack-ci-puppet - # /blob/master/modules/gerrit/manifests/init.pp#L74 - #Changeid|bug|blueprint - git_keywords = (r'(I[0-9a-f]{8,40})|' - '([Bb]ug|[Ll][Pp])[\s\#:]*(\d+)|' - '([Bb]lue[Pp]rint|[Bb][Pp])[\s\#:]*([A-Za-z0-9\\-]+)') - GIT_REGEX = re.compile(git_keywords) - - error = False - #NOTE(jogo) if match regex but over 3 words, acceptable title - if GIT_REGEX.search(title) is not None and len(title.split()) <= 3: - print ("N801: git commit title ('%s') should provide an accurate " - "description of the change, not just a reference to a bug " - "or blueprint" % title.strip()) - error = True - # HACKING.rst recommends commit titles 50 chars or less, but enforces - # a 72 character limit - if len(title.decode('utf-8')) > 72: - print ("N802: git commit title ('%s') should be under 50 chars" - % title.strip()) - error = True - return error - -imports_on_separate_lines_N301_compliant = r""" - Imports should usually be on separate lines. - - Okay: import os\nimport sys - E401: import sys, os - - N301: from subprocess import Popen, PIPE - Okay: from myclas import MyClass - Okay: from foo.bar.yourclass import YourClass - Okay: import myclass - Okay: import foo.bar.yourclass - """ - -if __name__ == "__main__": - #include nova path - sys.path.append(os.getcwd()) - #Run once tests (not per line) - once_error = once_git_check_commit_title() - #NOVA error codes start with an N - pep8.SELFTEST_REGEX = re.compile(r'(Okay|[EWN]\d{3}):\s(.*)') - pep8.ERRORCODE_REGEX = re.compile(r'[EWN]\d{3}') - add_nova() - pep8.current_file = current_file - pep8.readlines = readlines - pep8.StyleGuide.excluded = excluded - pep8.StyleGuide.input_dir = input_dir - # we need to kill this doctring otherwise the self tests fail - pep8.imports_on_separate_lines.__doc__ = \ - imports_on_separate_lines_N301_compliant - - try: - pep8._main() - sys.exit(once_error) - finally: - if len(_missingImport) > 0: - print >> sys.stderr, ("%i imports missing in this test environment" - % len(_missingImport)) diff --git a/tools/install_venv b/tools/install_venv index 96c9efd1..d872f53d 100755 --- a/tools/install_venv +++ b/tools/install_venv @@ -1,155 +1,3 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 +#!/bin/sh -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Copyright 2010 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Installation script for setting up development virtualenv -""" - -import os -import subprocess -import sys - - -ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -VENV = os.path.join(ROOT, '.venv') -PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') -TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires') - - -def die(message, *args): - print >> sys.stderr, message % args - sys.exit(1) - - -def run_command(cmd, redirect_output=True, check_exit_code=True): - """ - Runs a command in an out-of-process shell, returning the - output of that command. Working directory is ROOT. - """ - if redirect_output: - stdout = subprocess.PIPE - else: - stdout = None - - proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) - output = proc.communicate()[0] - if check_exit_code and proc.returncode != 0: - die('Command "%s" failed.\n%s', ' '.join(cmd), output) - return output - - -HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], - check_exit_code=False).strip()) -HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], - check_exit_code=False).strip()) - - -def check_dependencies(): - """Make sure virtualenv is in the path.""" - - if not HAS_VIRTUALENV: - print 'not found.' - # Try installing it via easy_install... - if HAS_EASY_INSTALL: - print 'Installing virtualenv via easy_install...', - if not run_command(['which', 'easy_install']): - die('ERROR: virtualenv not found.\n\n' - 'Virtualenv is required for development, please install' - ' it using your favorite package management tool') - print 'done.' - print 'done.' - - -def create_virtualenv(venv=VENV): - """ - Creates the virtual environment and installs PIP only into the - virtual environment - """ - print 'Creating venv...', - run_command(['virtualenv', '-q', '--no-site-packages', VENV]) - print 'done.' - print 'Installing pip in virtualenv...', - if not run_command(['tools/with_venv', 'easy_install', - 'pip>1.0']).strip(): - die("Failed to install pip.") - print 'done.' - - -def pip_install(*args): - run_command(['tools/with_venv', - 'pip', 'install', '--upgrade'] + list(args), - redirect_output=False) - - -def install_dependencies(venv=VENV): - print 'Installing dependencies with pip (this can take a while)...' - - pip_install('pip') - - pip_install('-r', PIP_REQUIRES) - pip_install('-r', TEST_REQUIRES) - - # Tell the virtual env how to "import app" - py_ver = _detect_python_version(venv) - pthfile = os.path.join(venv, "lib", py_ver, - "site-packages", "savanna.pth") - f = open(pthfile, 'w') - f.write("%s\n" % ROOT) - - -def _detect_python_version(venv): - lib_dir = os.path.join(venv, "lib") - for pathname in os.listdir(lib_dir): - if pathname.startswith('python'): - return pathname - raise Exception('Unable to detect Python version') - - -def print_help(): - help = """ - Development environment setup is complete. - - It uses virtualenv to track and manage Python dependencies - while in development and testing. - - To activate the development virtualenv for the extent of your current shell session - you can run: - - $ source .venv/bin/activate - - Or, if you prefer, you can run commands in the virtualenv on a case by case - basis by running: - - $ tools/with_venv - - Also, build_docs and run_tests will automatically use the virtualenv. - """ - print help - - -def main(argv): - check_dependencies() - create_virtualenv() - install_dependencies() - print_help() - -if __name__ == '__main__': - main(sys.argv) +tox -vvv -epep8 -- python --version diff --git a/tools/run_fast_checks b/tools/run_fast_checks index 65d1546f..14686b22 100755 --- a/tools/run_fast_checks +++ b/tools/run_fast_checks @@ -1,3 +1,3 @@ #!/bin/sh -tools/run_pep8 && tools/run_pyflakes +tools/run_pep8 diff --git a/tools/run_pep8 b/tools/run_pep8 index 3dc79979..256ec835 100755 --- a/tools/run_pep8 +++ b/tools/run_pep8 @@ -1,26 +1,3 @@ #!/bin/sh -set -e - -if [ ${VIRTUAL_ENV:+x} ] - then echo "VIRTUAL_ENV is already specified" - else export VIRTUAL_ENV=.venv -fi - -echo "VIRTUAL_ENV = $VIRTUAL_ENV" - -echo "Running pep8/hacking.py checks" -echo "==============================" - -# python tools/hacking.py --doctest - -# Until all these issues get fixed, ignore. -PEP8="$VIRTUAL_ENV/bin/python tools/hacking.py --ignore=N301" - -EXCLUDE="--exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*" -EXCLUDE=$EXCLUDE:",*egg,build,*hacking.py" -${PEP8} ${EXCLUDE} . - -${PEP8} --filename=savanna* bin - -! $VIRTUAL_ENV/bin/pyflakes savanna/ | grep "imported but unused\|redefinition of function" +tox -epep8 diff --git a/tools/run_pyflakes b/tools/run_pyflakes index 58b23c1d..30593ede 100755 --- a/tools/run_pyflakes +++ b/tools/run_pyflakes @@ -1,13 +1,3 @@ #!/bin/sh -if [ ${VIRTUAL_ENV:+x} ] - then echo "VIRTUAL_ENV is already specified" - else export VIRTUAL_ENV=.venv -fi - -echo "VIRTUAL_ENV = $VIRTUAL_ENV" - -echo "Running PyFlakes checks" -echo "=======================" - -$VIRTUAL_ENV/bin/pyflakes bin bin/savanna-api bin/savanna-manage savanna setup.py +echo "Please, use tools/run_pep8 instead of tools/run_pyflakes" diff --git a/tools/run_pylint b/tools/run_pylint index 4929b0a6..c69b8d1f 100755 --- a/tools/run_pylint +++ b/tools/run_pylint @@ -1,13 +1,3 @@ #!/bin/sh -if [ ${VIRTUAL_ENV:+x} ] - then echo "VIRTUAL_ENV is already specified" - else export VIRTUAL_ENV=.venv -fi - -echo "VIRTUAL_ENV = $VIRTUAL_ENV" - -echo "Running PyLint checks" -echo "=====================" - -$VIRTUAL_ENV/bin/pylint --output-format=parseable --rcfile=.pylintrc bin/savanna-api bin/savanna-manage savanna | tee pylint-report.txt +tox -epylint diff --git a/tools/run_tests b/tools/run_tests index 447ccfc6..ac5e6bfe 100755 --- a/tools/run_tests +++ b/tools/run_tests @@ -1,22 +1,3 @@ #!/bin/bash -if [ ${VIRTUAL_ENV:+x} ] - then echo "VIRTUAL_ENV is already specified" - else export VIRTUAL_ENV=.venv -fi - -echo "VIRTUAL_ENV = $VIRTUAL_ENV" - -echo "Running tests" -echo "=============" - -export NOSE_WITH_OPENSTACK=1 -export NOSE_OPENSTACK_COLOR=1 -export NOSE_OPENSTACK_RED=0.05 -export NOSE_OPENSTACK_YELLOW=0.025 -export NOSE_OPENSTACK_SHOW_ELAPSED=1 -export NOSE_OPENSTACK_STDOUT=1 -export NOSE_XUNIT=1 - -find . -type f -name "*.pyc" -delete -$VIRTUAL_ENV/bin/nosetests $@ +tox -epy27 diff --git a/tools/test-requires b/tools/test-requires index b970ed22..1c7abe6d 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,6 +1,6 @@ -pep8>=1.3.3 +flake8 +hacking pylint -pyflakes sphinx>=1.1.2 sphinxcontrib-httpdomain>=1.1.7 nose diff --git a/tools/with_venv b/tools/with_venv deleted file mode 100755 index e07f4499..00000000 --- a/tools/with_venv +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -TOOLS=`dirname $0` -VENV=$TOOLS/../.venv -source $VENV/bin/activate && $@ diff --git a/tox.ini b/tox.ini index b5bce5b7..8e6ea3a1 100644 --- a/tox.ini +++ b/tox.ini @@ -2,27 +2,53 @@ envlist = py26,py27,pep8,pyflakes [testenv] -setenv = VIRTUAL_ENV={envdir} +setenv = + VIRTUAL_ENV={envdir} + NOSE_WITH_OPENSTACK=1 + NOSE_OPENSTACK_COLOR=1 + NOSE_OPENSTACK_RED=0.05 + NOSE_OPENSTACK_YELLOW=0.025 + NOSE_OPENSTACK_SHOW_ELAPSED=1 + NOSE_OPENSTACK_STDOUT=1 + NOSE_XUNIT=1 deps = - -r{toxinidir}/tools/pip-requires - -r{toxinidir}/tools/test-requires -commands = bash tools/run_tests {posargs} + -r{toxinidir}/tools/pip-requires + -r{toxinidir}/tools/test-requires +commands = nosetests {posargs} + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 [tox:jenkins] downloadcache = ~/cache/pip [testenv:pep8] deps = - pep8>=1.3.3 - pyflakes -commands = bash tools/run_pep8 - -[testenv:cover] -setenv = NOSE_WITH_COVERAGE=1 + flake8 + hacking +commands = flake8 [testenv:pyflakes] -deps = {[testenv]deps} -commands = bash tools/run_pyflakes +deps = +commands = [testenv:venv] commands = {posargs} + +[testenv:docs] +commands = + rm -rf doc/html doc/build + rm -rf doc/source/apidoc doc/source/api + python setup.py build_sphinx + +[testenv:pylint] +commands = + pylint --output-format=parseable --rcfile=.pylintrc bin/savanna-api bin/savanna-manage savanna | tee pylint-report.txt + +[flake8] +# H301 one import per line +# H302 import only modules +ignore = H301,H302 +show-source = true +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,tools