Add .gitreview, setup.py and other infrastructure.

* .gitreview
* Disutils files
* tox configuration
* add .venv to ignore
* "mistral" package
* Sphinx documentation skeleton

Change-Id: I1d9aa5105b0dbc458665e9b16fd84826a18a36bf

.gitignore

@@ -8,6 +8,7 @@
 *.egg-info
 dist
 build
+.venv
 eggs
 parts
 bin
@@ -35,3 +36,5 @@ nosetests.xml
 .pydevproject
 .idea
 etc/mistral.conf
+tools/lintstack.head.py
+tools/pylint_exceptions

.gitreview

@@ -0,0 +1,4 @@
+[gerrit]
+host=review.openstack.org
+port=29418
+project=stackforge/mistral.git

.pylintrc

@@ -0,0 +1,49 @@
+# The format of this file isn't really documented; just use --generate-rcfile
+[MASTER]
+# Add <file or directory> to the black list. It should be a base name, not a
+# path. You may set this option multiple times.
+ignore=tests
+ignore=openstack
+
+[Messages Control]
+# NOTE(justinsb): We might want to have a 2nd strict pylintrc in future
+# C0111: Don't require docstrings on every method
+# W0511: TODOs in code comments are fine.
+# W0142: *args and **kwargs are fine.
+# W0622: Redefining id is fine.
+# W0703: Catch "Exception".
+disable=C0111,W0511,W0142,W0622,W0703
+
+[Basic]
+# Variable names can be 1 to 31 characters long, with lowercase and underscores
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Argument names can be 2 to 31 characters long, with lowercase and underscores
+argument-rgx=[a-z_][a-z0-9_]{1,30}$
+
+# Type attributes names can be 2 to 31 characters long, with lowercase and underscores
+attr-rgx=[a-z_][a-z0-9_]{1,30}$
+
+# Method names shold be at least 3 characters long and be lowercased with underscores
+method-rgx=([a-z_][a-z0-9_]{1,30}|setUp|tearDown)$
+
+# Module names matching savanna-* are ok (files in bin/)
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|(savanna-[a-z0-9_-]+))$
+
+# Don't require docstrings on tests.
+no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$
+
+[Design]
+max-public-methods=100
+min-public-methods=0
+max-args=6
+
+[Variables]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+# _ is used by our localization
+additional-builtins=_
+
+[TYPECHECK]
+generated-members=query,node_template,status_code,data

doc/source/conf.py

@@ -0,0 +1,245 @@
+# -*- coding: utf-8 -*-
+#
+# Mistral documentation build configuration file, created by
+# sphinx-quickstart on Fri Nov  1 02:06:28 2013.
+#
+# This file is execfile()d with the current directory set to its containing
+# dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Mistral'
+copyright = u'2013, OpenStack Foundation'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.01'
+# The full version, including alpha/beta/rc tags.
+release = '0.01'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Mistraldoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author,
+#  documentclass [howto/manual]).
+latex_documents = [
+    ('index', 'Mistral.tex', u'Mistral Documentation',
+     u'OpenStack Foundation', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'mistral', u'Mistral Documentation',
+     [u'OpenStack Foundation'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    ('index', 'Mistral', u'Mistral Documentation',
+     u'OpenStack Foundation', 'Mistral', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'

doc/source/index.rst

@@ -0,0 +1,22 @@
+.. Mistral documentation master file, created by
+   sphinx-quickstart on Fri Nov  1 02:06:28 2013.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Mistral's documentation!
+===================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

mistral/version.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pbr import version
+
+version_info = version.VersionInfo('Mistral')

requirements.txt

@@ -0,0 +1 @@
+pbr>=0.5.21,<1.0

setup.cfg

@@ -0,0 +1,27 @@
+[metadata]
+name = mistral
+version = 0.01
+summary = Mistral Project
+description-file = README.rd
+#license = Apache Software License
+classifiers =
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+    Environment :: OpenStack
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    #License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+author = OpenStack
+author-email = openstack-dev@lists.openstack.org
+
+[files]
+packages =
+    mistral
+
+[nosetests]
+cover-package = mistral
+
+[extract_messages]
+keywords = _ gettext ngettext l_ lazy_gettext

setup.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+setuptools.setup(
+    name="mistral")

test-requirements.txt

@@ -0,0 +1,5 @@
+pep8==1.4.5
+pyflakes>=0.7.2,<0.7.4
+flake8==2.0
+pylint==0.25.2
+sphinx>=1.1.2

tools/install_venv

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+tox -vvv -evenv -- python --version

tools/lintstack.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2012, AT&T Labs, Yun Mao <yunmao@gmail.com>
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Stolen from OpenStack Nova
+
+"""pylint error checking."""
+
+import cStringIO as StringIO
+import json
+import re
+import sys
+
+from pylint import lint
+from pylint.reporters import text
+
+# Note(maoy): E1103 is error code related to partial type inference
+ignore_codes = ["E1103"]
+# Note(maoy): the error message is the pattern of E0202. It should be ignored
+# for savanna.tests modules
+ignore_messages = ["An attribute affected in savanna.tests"]
+# We ignore all errors in openstack.common because it should be checked 
+# elsewhere.
+ignore_modules = ["savanna/openstack/common/"]
+
+KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"
+
+
+class LintOutput(object):
+
+    _cached_filename = None
+    _cached_content = None
+
+    def __init__(self, filename, lineno, line_content, code, message,
+                 lintoutput):
+        self.filename = filename
+        self.lineno = lineno
+        self.line_content = line_content
+        self.code = code
+        self.message = message
+        self.lintoutput = lintoutput
+
+    @classmethod
+    def from_line(cls, line):
+        m = re.search(r"(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line)
+        matched = m.groups()
+        filename, lineno, code, message = (matched[0], int(matched[1]),
+                                           matched[2], matched[-1])
+        if cls._cached_filename != filename:
+            with open(filename) as f:
+                cls._cached_content = list(f.readlines())
+                cls._cached_filename = filename
+        line_content = cls._cached_content[lineno - 1].rstrip()
+        return cls(filename, lineno, line_content, code, message,
+                   line.rstrip())
+
+    @classmethod
+    def from_msg_to_dict(cls, msg):
+        """From the output of pylint msg, to a dict, where each key
+        is a unique error identifier, value is a list of LintOutput
+        """
+        result = {}
+        for line in msg.splitlines():
+            obj = cls.from_line(line)
+            if obj.is_ignored():
+                continue
+            key = obj.key()
+            if key not in result:
+                result[key] = []
+            result[key].append(obj)
+        return result
+
+    def is_ignored(self):
+        if self.code in ignore_codes:
+            return True
+        if any(self.filename.startswith(name) for name in ignore_modules):
+            return True
+        if any(msg in self.message for msg in ignore_messages):
+            return True
+        return False
+
+    def key(self):
+        if self.code in ["E1101", "E1103"]:
+            # These two types of errors are like Foo class has no member bar.
+            # We discard the source code so that the error will be ignored
+            # next time another Foo.bar is encountered.
+            return self.message, ""
+        return self.message, self.line_content.strip()
+
+    def json(self):
+        return json.dumps(self.__dict__)
+
+    def review_str(self):
+        return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n"
+                "%(code)s: %(message)s" % self.__dict__)
+
+
+class ErrorKeys(object):
+
+    @classmethod
+    def print_json(cls, errors, output=sys.stdout):
+        print >>output, "# automatically generated by tools/lintstack.py"
+        for i in sorted(errors.keys()):
+            print >>output, json.dumps(i)
+
+    @classmethod
+    def from_file(cls, filename):
+        keys = set()
+        for line in open(filename):
+            if line and line[0] != "#":
+                d = json.loads(line)
+                keys.add(tuple(d))
+        return keys
+
+
+def run_pylint():
+    buff = StringIO.StringIO()
+    reporter = text.ParseableTextReporter(output=buff)
+    args = ["--include-ids=y", "-E", "savanna"]
+    lint.Run(args, reporter=reporter, exit=False)
+    val = buff.getvalue()
+    buff.close()
+    return val
+
+
+def generate_error_keys(msg=None):
+    print "Generating", KNOWN_PYLINT_EXCEPTIONS_FILE
+    if msg is None:
+        msg = run_pylint()
+    errors = LintOutput.from_msg_to_dict(msg)
+    with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
+        ErrorKeys.print_json(errors, output=f)
+
+
+def validate(newmsg=None):
+    print "Loading", KNOWN_PYLINT_EXCEPTIONS_FILE
+    known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
+    if newmsg is None:
+        print "Running pylint. Be patient..."
+        newmsg = run_pylint()
+    errors = LintOutput.from_msg_to_dict(newmsg)
+
+    print "Unique errors reported by pylint: was %d, now %d." \
+           % (len(known), len(errors))
+    passed = True
+    for err_key, err_list in errors.items():
+        for err in err_list:
+            if err_key not in known:
+                print err.lintoutput
+                print
+                passed = False
+    if passed:
+        print "Congrats! pylint check passed."
+        redundant = known - set(errors.keys())
+        if redundant:
+            print "Extra credit: some known pylint exceptions disappeared."
+            for i in sorted(redundant):
+                print json.dumps(i)
+            print "Consider regenerating the exception file if you will."
+    else:
+        print ("Please fix the errors above. If you believe they are false"
+              " positives, run 'tools/lintstack.py generate' to overwrite.")
+        sys.exit(1)
+
+
+def usage():
+    print """Usage: tools/lintstack.py [generate|validate]
+    To generate pylint_exceptions file: tools/lintstack.py generate
+    To validate the current commit: tools/lintstack.py
+    """
+
+
+def main():
+    option = "validate"
+    if len(sys.argv) > 1:
+        option = sys.argv[1]
+    if option == "generate":
+        generate_error_keys()
+    elif option == "validate":
+        validate()
+    else:
+        usage()
+
+
+if __name__ == "__main__":
+    main()

tools/lintstack.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2012-2013, AT&T Labs, Yun Mao <yunmao@gmail.com>
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Stolen from OpenStack Nova
+
+# Use lintstack.py to compare pylint errors.
+# We run pylint twice, once on HEAD, once on the code before the latest
+# commit for review.
+set -e
+TOOLS_DIR=$(cd $(dirname "$0") && pwd)
+# Get the current branch name.
+GITHEAD=`git rev-parse --abbrev-ref HEAD`
+if [[ "$GITHEAD" == "HEAD" ]]; then
+    # In detached head mode, get revision number instead
+    GITHEAD=`git rev-parse HEAD`
+    echo "Currently we are at commit $GITHEAD"
+else
+    echo "Currently we are at branch $GITHEAD"
+fi
+
+cp -f $TOOLS_DIR/lintstack.py $TOOLS_DIR/lintstack.head.py
+
+if git rev-parse HEAD^2 2>/dev/null; then
+    # The HEAD is a Merge commit. Here, the patch to review is
+    # HEAD^2, the master branch is at HEAD^1, and the patch was
+    # written based on HEAD^2~1.
+    PREV_COMMIT=`git rev-parse HEAD^2~1`
+    git checkout HEAD~1
+    # The git merge is necessary for reviews with a series of patches.
+    # If not, this is a no-op so won't hurt either.
+    git merge $PREV_COMMIT
+else
+    # The HEAD is not a merge commit. This won't happen on gerrit.
+    # Most likely you are running against your own patch locally.
+    # We assume the patch to examine is HEAD, and we compare it against
+    # HEAD~1
+    git checkout HEAD~1
+fi
+
+# First generate tools/pylint_exceptions from HEAD~1
+$TOOLS_DIR/lintstack.head.py generate
+# Then use that as a reference to compare against HEAD
+git checkout $GITHEAD
+$TOOLS_DIR/lintstack.head.py
+echo "Check passed. FYI: the pylint exceptions are:"
+cat $TOOLS_DIR/pylint_exceptions
+

tools/run_pep8

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+tox -epep8

tools/run_pylint

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+tox -epylint

tox.ini

@@ -0,0 +1,42 @@
+[tox]
+envlist = py26,py27,py33,pep8
+minversion = 1.6
+skipsdist = True
+
+[testenv]
+usedevelop = True
+install_command = pip install -U {opts} {packages}
+setenv =
+    VIRTUAL_ENV={envdir}
+    NOSE_WITH_OPENSTACK=1
+    NOSE_OPENSTACK_COLOR=1
+    NOSE_OPENSTACK_RED=0.05
+    NOSE_OPENSTACK_YELLOW=0.025
+    NOSE_OPENSTACK_SHOW_ELAPSED=1
+    NOSE_OPENSTACK_STDOUT=1
+    NOSE_XUNIT=1
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+
+[testenv:pep8]
+commands = flake8 {posargs}
+
+[testenv:venv]
+commands = {posargs}
+
+[testenv:docs]
+commands =
+    rm -rf doc/html doc/build
+    rm -rf doc/source/apidoc doc/source/api
+    python setup.py build_sphinx
+
+[testenv:pylint]
+setenv = VIRTUAL_ENV={envdir}
+commands = bash tools/lintstack.sh
+
+[flake8]
+show-source = true
+builtins = _
+exclude=.venv,.git,.tox,*egg,tools
+