openstack/taskflow
Code Issues Proposed changes

Use the released toxgen tool instead of our copy

Browse Source 
Update the docs to explain how to use the toxgen.py
released on pypi and use it to regenerate the tox.ini
file.

Change-Id: I9e0d28f0e40aa0280132a644260d028190b10974
This commit is contained in:
Joshua Harlow 2014-01-12 08:01:50 -08:00
parent 135f562408
commit 28fa164fcd
4 changed files with 128 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -14,12 +14,12 @@ Join us
Help
----
### Tox.ini
**Tox.ini**
To generate tox.ini, use the `toxgen.py` tool located in `tools/` and provide
that script as input the `tox-tmpl.ini` file to generate the final `tox.ini`
file.
To generate tox.ini, use the `toxgen.py` script by first installing
[toxgen](https://pypi.python.org/pypi/toxgen/) and then provide that script
as input the `tox-tmpl.ini` file to generate the final `tox.ini` file.
For example:
*For example:*
$ ./tools/toxgen.py -i tox-tmpl.ini -o tox.ini
$ toxgen.py -i tox-tmpl.ini -o tox.ini

211
tools/toxgen.py
View File

@ -1,211 +0,0 @@
#!/usr/bin/env python
# From: https://bitbucket.org/cdevienne/toxgen (pypi soon hopefully) and
# modified slightly to work in python 2.6 and set some values that are not
# being set.
#
# TODO(harlowja): remove me when toxgen is a pypi package.
"""
Produce a tox.ini file from a template config file.
The template config file is a standard tox.ini file with additional sections.
Theses sections will be combined to create new testenv: sections if they do
not exists yet.
"""
import collections
import itertools
import optparse
import os
import six
from six.moves import configparser
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
HEADER = '# DO NOT EDIT THIS FILE - it is machine generated from %(filename)s'
SKIP_VENVS = frozenset(['venv'])
parser = optparse.OptionParser(epilog=__doc__)
parser.add_option('-i', '--input', dest='input',
default='tox-tmpl.ini', metavar='FILE')
parser.add_option('-o', '--output', dest='output',
default='tox.ini', metavar='FILE')
class AxisItem(object):
def __init__(self, axis, name, config):
self.axis = axis
self.isdefault = name[-1] == '*'
self.name = name[:-1] if self.isdefault else name
self.load(config)
def load(self, config):
sectionname = 'axis:%s:%s' % (self.axis.name, self.name)
if config.has_section(sectionname):
self.options = dict(config.items(sectionname))
else:
self.options = dict()
for name, value in self.axis.defaults.items():
if name not in self.options:
self.options[name] = value
class Axis(object):
def __init__(self, name, config):
self.name = name
self.load(config)
def load(self, config):
self.items = dict()
values = config.get('axes', self.name).split(',')
if config.has_section('axis:%s' % self.name):
self.defaults = dict(
config.items('axis:%s' % self.name)
)
else:
self.defaults = {}
for value in values:
self.items[value.strip('*')] = AxisItem(self, value, config)
def format_list(contents, max_len=80, sep=","):
lines = []
for line in contents:
if not lines:
lines.append(line + ",")
else:
last_len = len(lines[-1])
if last_len + len(line) >= max_len:
lines.append(str(line) + sep)
else:
lines[-1] = lines[-1] + str(line) + sep
return "\n".join(lines).rstrip(",")
def render(incfg, filename, adjust_envlist=True):
test_envs = set()
for s in incfg.sections():
if s.startswith("testenv:"):
env = s[len("testenv:"):].strip()
if env in SKIP_VENVS or not env:
continue
test_envs.add(env)
test_envs = [s for s in test_envs if s]
try:
envlist = incfg.get("tox", 'envlist')
envlist = [e.strip() for e in envlist.split(",")]
envlist = set([e for e in envlist if e])
except (configparser.NoOptionError, configparser.NoSectionError):
envlist = set()
for e in test_envs:
if e not in envlist:
envlist.add(e)
if not incfg.has_section("tox"):
incfg.add_section("tox")
incfg.set("tox", "envlist",
format_list(list(sorted(envlist)), max_len=-1))
text = six.StringIO()
incfg.write(text)
contents = [
HEADER % {'filename': os.path.basename(filename)},
'',
# Remove how configparser uses tabs instead of spaces, madness...
text.getvalue().replace("\t", " " * 4),
]
return "\n".join(contents)
def compile_template(incfg):
axes = dict()
if incfg.has_section('axes'):
for axis in incfg.options('axes'):
axes[axis] = Axis(axis, incfg)
out = configparser.ConfigParser(dict_type=OrderedDict)
for section in incfg.sections():
if section == 'axes' or section.startswith('axis:'):
continue
out.add_section(section)
for name, value in incfg.items(section):
out.set(section, name, value)
items = [axis.items.keys() for axis in axes.values()]
for combination in itertools.product(*items):
options = {}
section_name = (
'testenv:' + '-'.join([item for item in combination if item])
)
section_alt_name = (
'testenv:' + '-'.join([
itemname
for axis, itemname in zip(axes.values(), combination)
if itemname and not axis.items[itemname].isdefault
])
)
if section_alt_name == section_name:
section_alt_name = None
axes_items = [
'%s:%s' % (axis, itemname)
for axis, itemname in zip(axes, combination)
]
for axis, itemname in zip(axes.values(), combination):
axis_options = axis.items[itemname].options
if 'constraints' in axis_options:
constraints = axis_options['constraints'].split('\n')
for c in constraints:
if c.startswith('!') and c[1:] in axes_items:
continue
for name, value in axis_options.items():
if name in options:
options[name] += value
else:
options[name] = value
constraints = options.pop('constraints', '').split('\n')
neg_constraints = [c[1:] for c in constraints if c and c[0] == '!']
if not set(neg_constraints).isdisjoint(axes_items):
continue
if not out.has_section(section_name):
out.add_section(section_name)
if (section_alt_name and not out.has_section(section_alt_name)):
out.add_section(section_alt_name)
for name, value in reversed(options.items()):
if not out.has_option(section_name, name):
out.set(section_name, name, value)
if section_alt_name and not out.has_option(section_alt_name, name):
out.set(section_alt_name, name, value)
return out
def main():
options, args = parser.parse_args()
tmpl = configparser.ConfigParser()
with open(options.input, 'rb') as fh:
tmpl.readfp(fh, filename=options.input)
with open(options.output, 'wb') as outfile:
text = render(compile_template(tmpl), options.input)
outfile.write(text)
if __name__ == '__main__':
main()

3
tox-tmpl.ini
View File

@ -21,8 +21,7 @@ commands = python setup.py testr --slowest --testr-args='{posargs}'
downloadcache = ~/cache/pip
[testenv:pep8]
commands =
flake8 {posargs}
commands = flake8 {posargs}
[testenv:pylint]
setenv = VIRTUAL_ENV={envdir}

243
tox.ini
View File

@ -28,33 +28,32 @@ envlist = cover,
pylint
[testenv]
usedevelop = True
install_command = pip install {opts} {packages}
commands = python setup.py testr --slowest --testr-args='{posargs}'
setenv = VIRTUAL_ENV={envdir}
LANG=en_US.UTF-8
LANGUAGE=en_US:en
LC_ALL=C
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
usedevelop = True
[testenv:pylint]
commands = pylint
setenv = VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/requirements.txt
pylint==0.26.0
[testenv:cover]
commands = python setup.py testr --coverage --testr-args='{posargs}'
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = python setup.py testr --slowest --testr-args='{posargs}'
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:pep8]
commands =
flake8 {posargs}
commands = flake8 {posargs}
[testenv:pylint]
setenv = VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/requirements.txt
pylint==0.26.0
commands = pylint
[testenv:cover]
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = python setup.py testr --coverage --testr-args='{posargs}'
[testenv:venv]
commands = {posargs}
@ -64,131 +63,131 @@ ignore = H402
builtins = _
exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools
[testenv:py27]
basepython = python2.7
deps = {[testenv]deps}
[testenv:py27-ev]
basepython = python2.7
deps = {[testenv]deps}
eventlet>=0.13.0
[testenv:py27-sa9]
basepython = python2.7
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
[testenv:py27-sa9-ev]
basepython = python2.7
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
eventlet>=0.13.0
[testenv:py27-sa8]
basepython = python2.7
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
[testenv:py27-sa8-ev]
basepython = python2.7
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
eventlet>=0.13.0
[testenv:py27-sa7]
basepython = python2.7
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
[testenv:py27-sa7-ev]
basepython = python2.7
[testenv:py26-sa7-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
eventlet>=0.13.0
[testenv:py26]
basepython = python2.6
deps = {[testenv]deps}
[testenv:py26-ev]
basepython = python2.6
deps = {[testenv]deps}
eventlet>=0.13.0
[testenv:py26-sa9]
basepython = python2.6
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
[testenv:py26-sa9-ev]
basepython = python2.6
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
eventlet>=0.13.0
[testenv:py26-sa8]
basepython = python2.6
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
[testenv:py26-sa8-ev]
basepython = python2.6
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
eventlet>=0.13.0
[testenv:py26-sa7]
basepython = python2.6
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
[testenv:py26-sa7-ev]
basepython = python2.6
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
eventlet>=0.13.0
[testenv:py33]
basepython = python3.3
deps = {[testenv]deps}
[testenv:py33-sa9]
basepython = python3.3
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
[testenv:py33-sa8]
basepython = python3.3
[testenv:py26-sa8-ev]
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.6
[testenv:py33-sa7]
basepython = python3.3
[testenv:py26-sa8]
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
basepython = python2.6
[testenv:py26-sa9-ev]
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.6
[testenv:py26-sa9]
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
basepython = python2.6
[testenv:py26-ev]
deps = {[testenv]deps}
eventlet>=0.13.0
basepython = python2.6
[testenv:py26]
deps = {[testenv]deps}
basepython = python2.6
[testenv:py27-sa7-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa7]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
basepython = python2.7
[testenv:py27-sa8-ev]
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa8]
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
basepython = python2.7
[testenv:py27-sa9-ev]
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa9]
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
basepython = python2.7
[testenv:py27-ev]
deps = {[testenv]deps}
eventlet>=0.13.0
basepython = python2.7
[testenv:py27]
deps = {[testenv]deps}
basepython = python2.7
[testenv:py33-sa7]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
basepython = python3.3
[testenv:py33-sa8]
deps = {[testenv]deps}
SQLAlchemy>0.7.99
SQLAlchemy<=0.8.99
alembic>=0.4.1
basepython = python3.3
[testenv:py33-sa9]
deps = {[testenv]deps}
SQLAlchemy>0.8.99
SQLAlchemy<=0.9.99
alembic>=0.4.1
basepython = python3.3
[testenv:py33]
deps = {[testenv]deps}
basepython = python3.3