Support for running tools under python3

If we run the tools (create-directories or check-all-candidacies) under
python3 they'll fail with silly errors (like[1]).  This is because we
have a number of assumptions they we'll be running under python2 like:

 - pickle.protocol: we done specify one so if we write a pickle file
   from python3 we can't read it in python2.
 - urllib and urlparse: those modules change paths in python 2 or 3 so
   use the six helpers to get the bits we want.
 - Explicitly write files in binary format
 - Various str vs bytes issues
 - dict().keys() returns dict_keys under python3 explictly cast that to
   list()

After addressing these issues we can run the tools under either python2
or python3.  To make this easy to check create explicit ven2 and venv3
test environments, leaving all other testenvs to use the system default.

[1] http://paste.openstack.org/show/593823/
Change-Id: I20334a52500847c810b486c9c8b108e75a5d6303
This commit is contained in:
Tony Breeds 2017-01-04 16:28:24 +11:00
parent 5ddf83e255
commit a3ac4a90f8
8 changed files with 27 additions and 14 deletions

View File

@ -33,7 +33,7 @@ def render_list(list_type, candidates_list):
output_file = os.path.join(".", "doc", "source", "%s.rst" % list_type) output_file = os.path.join(".", "doc", "source", "%s.rst" % list_type)
template_name = "%s.jinja" % list_type template_name = "%s.jinja" % list_type
template_dir = os.path.join(".", "doc", "source", "_exts") template_dir = os.path.join(".", "doc", "source", "_exts")
with open(output_file, "w") as out: with open(output_file, "wb") as out:
out.write( out.write(
render_template( render_template(
template_name, template_name,
@ -59,7 +59,7 @@ def build_archive(serie, list_type):
output = os.path.join(".", "doc", "source", serie, "%s.rst" % list_type) output = os.path.join(".", "doc", "source", serie, "%s.rst" % list_type)
template_name = "%s_archive.jinja" % list_type template_name = "%s_archive.jinja" % list_type
template_dir = os.path.join(".", "doc", "source", "_exts") template_dir = os.path.join(".", "doc", "source", "_exts")
with open(output, "w") as out: with open(output, "wb") as out:
out.write( out.write(
render_template( render_template(
template_name, template_name,

View File

@ -134,7 +134,7 @@ html_static_path = ['_static']
git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local", git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
"-n1"] "-n1"]
html_last_updated_fmt = subprocess.Popen( html_last_updated_fmt = subprocess.Popen(
git_cmd, stdout=subprocess.PIPE).communicate()[0] git_cmd, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
# If true, SmartyPants will be used to convert quotes and dashes to # If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities. # typographically correct entities.

View File

@ -41,9 +41,10 @@ def main():
args.limit = 100 args.limit = 100
projects = utils.get_projects(tag=args.tag) projects = utils.get_projects(tag=args.tag)
if args.project_name not in projects.keys(): projects_list = list(projects.keys())
if args.project_name not in projects_list:
print('[E]: %s not found in: %s' % print('[E]: %s not found in: %s' %
(args.project_name, ','.join(projects.keys()))) (args.project_name, ','.join(projects_list)))
return 1 return 1
if check_candidacy.check_candidate(args.project_name, args.email, if check_candidacy.check_candidate(args.project_name, args.email,

View File

@ -30,7 +30,7 @@ def main():
exit(1) exit(1)
projects = utils.get_projects() projects = utils.get_projects()
project_list = projects.keys() project_list = list(projects.keys())
project_list.sort() project_list.sort()
for project in project_list + ["TC"]: for project in project_list + ["TC"]:
dpath = "%s/%s" % (base_dir, utils.name2dir(project)) dpath = "%s/%s" % (base_dir, utils.name2dir(project))

View File

@ -45,7 +45,7 @@ def collect_project_stats(basedir, verbose):
project = directory[len(basedir):] project = directory[len(basedir):]
if project == "TC": if project == "TC":
continue continue
candidates = filter(lambda x: x.endswith('.txt'), filenames) candidates = list(filter(lambda x: x.endswith('.txt'), filenames))
candidates_count = len(candidates) candidates_count = len(candidates)
if not filenames == []: if not filenames == []:

View File

@ -21,11 +21,14 @@ import pickle
import pytz import pytz
import re import re
import requests import requests
import six
import subprocess import subprocess
import time import time
import urllib
import yaml import yaml
from six.moves.urllib.parse import quote_plus
from six.moves.urllib.request import urlopen
from openstack_election import config from openstack_election import config
@ -110,7 +113,7 @@ def get_reviews(query):
opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS'] opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS']
opts_str = '&o=%s' % ('&o='.join(opts)) opts_str = '&o=%s' % ('&o='.join(opts))
url = ('%s/changes/?q=%s%s' % url = ('%s/changes/?q=%s%s' %
(GERRIT_BASE, urllib.quote_plus(query, safe='/:=><^.*'), opts_str)) (GERRIT_BASE, quote_plus(query, safe='/:=><^.*'), opts_str))
return gerrit_query(url) return gerrit_query(url)
@ -141,10 +144,10 @@ def get_projects(tag=None):
os.stat(cache_file).st_size < 100 or os.stat(cache_file).st_size < 100 or
os.stat(cache_file).st_mtime + (7*24*3600) < time.time()): os.stat(cache_file).st_mtime + (7*24*3600) < time.time()):
print("[+] Updating %s" % (cache_file)) print("[+] Updating %s" % (cache_file))
data = yaml.safe_load(urllib.urlopen(url).read()) data = yaml.safe_load(urlopen(url).read())
pickle.dump(data, open(cache_file, "w")) pickle.dump(data, open(cache_file, "wb"), protocol=2)
return pickle.load(open(cache_file)) return pickle.load(open(cache_file, "rb"))
# Election functions # Election functions
@ -176,7 +179,7 @@ def build_candidates_list(election=conf['release']):
project_prefix = os.path.join(CANDIDATE_PATH, election, project) project_prefix = os.path.join(CANDIDATE_PATH, election, project)
file_list = filter( file_list = filter(
lambda x: x.endswith(".txt"), lambda x: x.endswith(".txt"),
os.listdir(unicode(project_prefix)), os.listdir(project_prefix),
) )
candidates_list = [] candidates_list = []
for candidate_file in file_list: for candidate_file in file_list:
@ -185,7 +188,7 @@ def build_candidates_list(election=conf['release']):
{ {
'url': ('%s/%s/plain/%s' % 'url': ('%s/%s/plain/%s' %
(CGIT_URL, ELECTION_REPO, (CGIT_URL, ELECTION_REPO,
urllib.quote_plus(filepath, safe='/'))), quote_plus(filepath, safe='/'))),
'ircname': candidate_file[:-4].replace('`', r'\`'), 'ircname': candidate_file[:-4].replace('`', r'\`'),
'email': get_email(filepath), 'email': get_email(filepath),
'fullname': get_fullname(filepath) 'fullname': get_fullname(filepath)

View File

@ -1,6 +1,7 @@
# The order of packages is significant, because pip processes them in the order # The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration # of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
six>=1.9.0 # MIT
pytz>=2013.6 # MIT pytz>=2013.6 # MIT
PyYAML>=3.1.0 # MIT PyYAML>=3.1.0 # MIT
requests>=2.10.0 # Apache-2.0 requests>=2.10.0 # Apache-2.0

View File

@ -21,6 +21,14 @@ commands = yamllint configuration.yaml
[testenv:venv] [testenv:venv]
commands = {posargs} commands = {posargs}
[testenv:venv2]
basepython = python2
commands = {posargs}
[testenv:venv3]
basepython = python3
commands = {posargs}
[testenv:docs] [testenv:docs]
commands = python setup.py build_sphinx commands = python setup.py build_sphinx