openstack
/
neutron
Code Issues Proposed changes
Browse Source

Add script for checking i18n message 

Part of bp make-string-localizable

usage: tox -e i18n

tools/check_i18n.py: used check i18n message for one file.

tools/check_i18n_test_case.txt: test case of check_i18n.py.
run test case with cmd:
$ ./tools/check_i18n.py ./tools/check_i18n_test_case.txt -d

Change-Id: I2c383b7bb11ab3bdb8e3bb3b887342b1225840ac
changes/01/17201/12
He Jie Xu 10 years ago
parent
2e043a5a78
commit
7ae701120a
4 changed files with 322 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 154
      tools/check_i18n.py
  2. 67
      tools/check_i18n_test_case.txt
  3. 98
      tools/i18n_cfg.py
  4. 3
      tox.ini

154
tools/check_i18n.py
Unescape View File

@ -0,0 +1,154 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import compiler
import imp
import os.path
import sys
def is_localized(node):
""" Check message wrapped by _() """
if isinstance(node.parent, compiler.ast.CallFunc):
if isinstance(node.parent.node, compiler.ast.Name):
if node.parent.node.name == '_':
return True
return False
class ASTWalker(compiler.visitor.ASTVisitor):
def default(self, node, *args):
for child in node.getChildNodes():
child.parent = node
compiler.visitor.ASTVisitor.default(self, node, *args)
class Visitor(object):
def __init__(self, filename, i18n_msg_predicates,
msg_format_checkers, debug):
self.filename = filename
self.debug = debug
self.error = 0
self.i18n_msg_predicates = i18n_msg_predicates
self.msg_format_checkers = msg_format_checkers
with open(filename) as f:
self.lines = f.readlines()
def visitConst(self, node):
if not isinstance(node.value, str):
return
if is_localized(node):
for (checker, msg) in self.msg_format_checkers:
if checker(node):
print >> sys.stderr, (
'%s:%d %s: %s' %
(self.filename, node.lineno,
self.lines[node.lineno - 1][:-1],
"Error: %s" % msg))
self.error = 1
return
if debug:
print ('%s:%d %s: %s' %
(self.filename, node.lineno,
self.lines[node.lineno - 1][:-1],
"Pass"))
else:
for (predicate, action, msg) in self.i18n_msg_predicates:
if predicate(node):
if action == 'skip':
if debug:
print ('%s:%d %s: %s' %
(self.filename, node.lineno,
self.lines[node.lineno - 1][:-1],
"Pass"))
return
elif action == 'error':
print >> sys.stderr, (
'%s:%d %s: %s' %
(self.filename, node.lineno,
self.lines[node.lineno - 1][:-1],
"Error: %s" % msg))
self.error = 1
return
elif action == 'warn':
print ('%s:%d %s: %s' %
(self.filename, node.lineno,
self.lines[node.lineno - 1][:-1],
"Warn: %s" % msg))
return
print >> sys.stderr, 'Predicate with wrong action!'
def is_file_in_black_list(black_list, f):
for f in black_list:
if os.path.abspath(input_file).startswith(
os.path.abspath(f)):
return True
return False
def check_i18n(input_file, i18n_msg_predicates, msg_format_checkers, debug):
input_mod = compiler.parseFile(input_file)
v = compiler.visitor.walk(input_mod,
Visitor(input_file,
i18n_msg_predicates,
msg_format_checkers,
debug),
ASTWalker())
return v.error
if __name__ == '__main__':
input_path = sys.argv[1]
cfg_path = sys.argv[2]
try:
cfg_mod = imp.load_source('', cfg_path)
except:
print >> sys.stderr, "Load cfg module failed"
sys.exit(1)
i18n_msg_predicates = cfg_mod.i18n_msg_predicates
msg_format_checkers = cfg_mod.msg_format_checkers
black_list = cfg_mod.file_black_list
debug = False
if len(sys.argv) > 3:
if sys.argv[3] == '-d':
debug = True
if os.path.isfile(input_path):
sys.exit(check_i18n(input_path,
i18n_msg_predicates,
msg_format_checkers,
debug))
error = 0
for dirpath, dirs, files in os.walk(input_path):
for f in files:
if not f.endswith('.py'):
continue
input_file = os.path.join(dirpath, f)
if is_file_in_black_list(black_list, input_file):
continue
if check_i18n(input_file,
i18n_msg_predicates,
msg_format_checkers,
debug):
error = 1
sys.exit(error)

67
tools/check_i18n_test_case.txt
Unescape View File

@ -0,0 +1,67 @@
# test-case for check_i18n.py
# python check_i18n.py check_i18n.txt -d
# message format checking
# capital checking
msg = _("hello world, error")
msg = _("hello world_var, error")
msg = _('file_list xyz, pass')
msg = _("Hello world, pass")
# format specifier checking
msg = _("Hello %s world %d, error")
msg = _("Hello %s world, pass")
msg = _("Hello %(var1)s world %(var2)s, pass")
# message has been localized
# is_localized
msg = _("Hello world, pass")
msg = _("Hello world, pass") % var
LOG.debug(_('Hello world, pass'))
LOG.info(_('Hello world, pass'))
raise x.y.Exception(_('Hello world, pass'))
raise Exception(_('Hello world, pass'))
# message need be localized
# is_log_callfunc
LOG.debug('hello world, error')
LOG.debug('hello world, error' % xyz)
sys.append('hello world, warn')
# is_log_i18n_msg_with_mod
LOG.debug(_('Hello world, error') % xyz)
# default warn
msg = 'hello world, warn'
msg = 'hello world, warn' % var
# message needn't be localized
# skip only one word
msg = ''
msg = "hello,pass"
# skip dict
msg = {'hello world, pass': 1}
# skip list
msg = ["hello world, pass"]
# skip subscript
msg['hello world, pass']
# skip xml marker
msg = "<test><t></t></test>, pass"
# skip sql statement
msg = "SELECT * FROM xyz WHERE hello=1, pass"
msg = "select * from xyz, pass"
# skip add statement
msg = 'hello world' + e + 'world hello, pass'
# skip doc string
"""
Hello world, pass
"""
class Msg:
pass

98
tools/i18n_cfg.py
Unescape View File

@ -0,0 +1,98 @@
import compiler
import re
def is_log_callfunc(n):
""" LOG.xxx('hello %s' % xyz) and LOG('hello') """
if isinstance(n.parent, compiler.ast.Mod):
n = n.parent
if isinstance(n.parent, compiler.ast.CallFunc):
if isinstance(n.parent.node, compiler.ast.Getattr):
if isinstance(n.parent.node.getChildNodes()[0],
compiler.ast.Name):
if n.parent.node.getChildNodes()[0].name == 'LOG':
return True
return False
def is_log_i18n_msg_with_mod(n):
""" LOG.xxx("Hello %s" % xyz) should be LOG.xxx("Hello %s", xyz) """
if not isinstance(n.parent.parent, compiler.ast.Mod):
return False
n = n.parent.parent
if isinstance(n.parent, compiler.ast.CallFunc):
if isinstance(n.parent.node, compiler.ast.Getattr):
if isinstance(n.parent.node.getChildNodes()[0],
compiler.ast.Name):
if n.parent.node.getChildNodes()[0].name == 'LOG':
return True
return False
def is_wrong_i18n_format(n):
""" Check _('hello %s' % xyz) """
if isinstance(n.parent, compiler.ast.Mod):
n = n.parent
if isinstance(n.parent, compiler.ast.CallFunc):
if isinstance(n.parent.node, compiler.ast.Name):
if n.parent.node.name == '_':
return True
return False
"""
Used for check message need be localized or not.
(predicate_func, action, message)
"""
i18n_msg_predicates = [
# Skip ['hello world', 1]
(lambda n: isinstance(n.parent, compiler.ast.List), 'skip', ''),
# Skip {'hellow world', 1}
(lambda n: isinstance(n.parent, compiler.ast.Dict), 'skip', ''),
# Skip msg['hello world']
(lambda n: isinstance(n.parent, compiler.ast.Subscript), 'skip', ''),
# Skip doc string
(lambda n: isinstance(n.parent, compiler.ast.Discard), 'skip', ''),
# Skip msg = "hello", in normal, message should more than one word
(lambda n: len(n.value.strip().split(' ')) <= 1, 'skip', ''),
# Skip msg = 'hello world' + vars + 'world hello'
(lambda n: isinstance(n.parent, compiler.ast.Add), 'skip', ''),
# Skip xml markers msg = "<test></test>"
(lambda n: len(re.compile("</.*>").findall(n.value)) > 0, 'skip', ''),
# Skip sql statement
(lambda n: len(
re.compile("^SELECT.*FROM", flags=re.I).findall(n.value)) > 0,
'skip', ''),
# LOG.xxx()
(is_log_callfunc, 'error', 'Message must be localized'),
# _('hello %s' % xyz) should be _('hello %s') % xyz
(is_wrong_i18n_format, 'error',
("Message format was wrong, _('hello %s' % xyz) "
"should be _('hello %s') % xyz")),
# default
(lambda n: True, 'warn', 'Message might need localized')
]
"""
Used for checking message format. (checker_func, message)
"""
msg_format_checkers = [
# If message contain more than on format specifier, it should use
# mapping key
(lambda n: len(re.compile("%[bcdeEfFgGnosxX]").findall(n.value)) > 1,
"The message shouldn't contain more than one format specifier"),
# Check capital
(lambda n: n.value.split(' ')[0].count('_') == 0 and
n.value[0].isalpha() and
n.value[0].islower(),
"First letter must be capital"),
(is_log_i18n_msg_with_mod,
'LOG.xxx("Hello %s" % xyz) should be LOG.xxx("Hello %s", xyz)')
]
file_black_list = ["./quantum/plugins/cisco/tests/unit",
"./quantum/tests/unit",
"./quantum/openstack",
"./quantum/plugins/bigswitch/tests"]

3
tox.ini
Unescape View File

@ -25,6 +25,9 @@ commands =
pep8 --repeat --show-source --ignore=E125 --exclude=.venv,.tox,dist,doc,openstack,*egg .
pep8 --repeat --show-source --ignore=E125 --filename=quantum* bin
[testenv:i18n]
commands = python ./tools/check_i18n.py ./quantum ./tools/i18n_cfg.py
[testenv:cover]
setenv = NOSE_WITH_COVERAGE=1

Write Preview
Loading…
Cancel
Save