Style checker for sphinx (or other) rst documentation.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

main.py 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342
  1. # Copyright (C) 2014 Ivan Melnikov <iv at altlinux dot org>
  2. #
  3. # Author: Joshua Harlow <harlowja@yahoo-inc.com>
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  6. # not use this file except in compliance with the License. You may obtain
  7. # a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14. # License for the specific language governing permissions and limitations
  15. # under the License.
  16. """Check documentation for simple style requirements.
  17. What is checked:
  18. - invalid rst format - D000
  19. - lines should not be longer than 79 characters - D001
  20. - RST exception: line with no whitespace except in the beginning
  21. - RST exception: lines with http or https urls
  22. - RST exception: literal blocks
  23. - RST exception: rst target directives
  24. - no trailing whitespace - D002
  25. - no tabulation for indentation - D003
  26. - no carriage returns (use unix newlines) - D004
  27. - no newline at end of file - D005
  28. """
  29. import argparse
  30. import collections
  31. import logging
  32. import os
  33. import sys
  34. if __name__ == '__main__':
  35. # Only useful for when running directly (for dev/debugging).
  36. sys.path.insert(0, os.path.abspath(os.getcwd()))
  37. sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.getcwd())))
  38. import six
  39. from six.moves import configparser
  40. from stevedore import extension
  41. from doc8 import checks
  42. from doc8 import parser as file_parser
  43. from doc8 import utils
  44. from doc8 import version
  45. FILE_PATTERNS = ['.rst', '.txt']
  46. MAX_LINE_LENGTH = 79
  47. CONFIG_FILENAMES = [
  48. "doc8.ini",
  49. "tox.ini",
  50. "pep8.ini",
  51. "setup.cfg",
  52. ]
  53. def split_set_type(text):
  54. return set([i.strip() for i in text.split(",") if i.strip()])
  55. def merge_sets(sets):
  56. m = set()
  57. for s in sets:
  58. m.update(s)
  59. return m
  60. def extract_config(args):
  61. parser = configparser.RawConfigParser()
  62. read_files = []
  63. if args['config']:
  64. for fn in args['config']:
  65. with open(fn, 'r') as fh:
  66. parser.readfp(fh, filename=fn)
  67. read_files.append(fn)
  68. else:
  69. read_files.extend(parser.read(CONFIG_FILENAMES))
  70. if not read_files:
  71. return {}
  72. cfg = {}
  73. try:
  74. cfg['max_line_length'] = parser.getint("doc8", "max-line-length")
  75. except (configparser.NoSectionError, configparser.NoOptionError):
  76. pass
  77. try:
  78. cfg['ignore'] = split_set_type(parser.get("doc8", "ignore"))
  79. except (configparser.NoSectionError, configparser.NoOptionError):
  80. pass
  81. try:
  82. cfg['ignore_path'] = split_set_type(parser.get("doc8",
  83. "ignore_path"))
  84. except (configparser.NoSectionError, configparser.NoOptionError):
  85. pass
  86. try:
  87. cfg['allow_long_titles'] = parser.getboolean("doc8",
  88. "allow-long-titles")
  89. except (configparser.NoSectionError, configparser.NoOptionError):
  90. pass
  91. try:
  92. cfg['sphinx'] = parser.getboolean("doc8", "sphinx")
  93. except (configparser.NoSectionError, configparser.NoOptionError):
  94. pass
  95. try:
  96. cfg['verbose'] = parser.getboolean("doc8", "verbose")
  97. except (configparser.NoSectionError, configparser.NoOptionError):
  98. pass
  99. try:
  100. cfg['file_encoding'] = parser.get("doc8", "file-encoding")
  101. except (configparser.NoSectionError, configparser.NoOptionError):
  102. pass
  103. try:
  104. cfg['default_extension'] = parser.get("doc8", "default-extension")
  105. except (configparser.NoSectionError, configparser.NoOptionError):
  106. pass
  107. try:
  108. extensions = parser.get("doc8", "extensions")
  109. extensions = extensions.split(",")
  110. extensions = [s.strip() for s in extensions if s.strip()]
  111. if extensions:
  112. cfg['extension'] = extensions
  113. except (configparser.NoSectionError, configparser.NoOptionError):
  114. pass
  115. return cfg
  116. def fetch_checks(cfg):
  117. base = [
  118. checks.CheckValidity(cfg),
  119. checks.CheckTrailingWhitespace(cfg),
  120. checks.CheckIndentationNoTab(cfg),
  121. checks.CheckCarriageReturn(cfg),
  122. checks.CheckMaxLineLength(cfg),
  123. checks.CheckNewlineEndOfFile(cfg),
  124. ]
  125. mgr = extension.ExtensionManager(
  126. namespace='doc8.extension.check',
  127. invoke_on_load=True,
  128. invoke_args=(cfg.copy(),),
  129. )
  130. addons = []
  131. for e in mgr:
  132. addons.append(e.obj)
  133. return base + addons
  134. def setup_logging(verbose):
  135. if verbose:
  136. level = logging.DEBUG
  137. else:
  138. level = logging.ERROR
  139. logging.basicConfig(level=level,
  140. format='%(levelname)s: %(message)s', stream=sys.stdout)
  141. def scan(cfg):
  142. print("Scanning...")
  143. files = collections.deque()
  144. ignored_paths = cfg.get('ignore_path', [])
  145. files_ignored = 0
  146. file_iter = utils.find_files(cfg.get('paths', []),
  147. cfg.get('extension', []), ignored_paths)
  148. default_extension = cfg.get('default_extension')
  149. file_encoding = cfg.get('file_encoding')
  150. for filename, ignoreable in file_iter:
  151. if ignoreable:
  152. files_ignored += 1
  153. if cfg.get('verbose'):
  154. print(" Ignoring '%s'" % (filename))
  155. else:
  156. f = file_parser.parse(filename,
  157. default_extension=default_extension,
  158. encoding=file_encoding)
  159. files.append(f)
  160. if cfg.get('verbose'):
  161. print(" Selecting '%s'" % (filename))
  162. return (files, files_ignored)
  163. def validate(cfg, files):
  164. print("Validating...")
  165. error_counts = {}
  166. ignoreables = frozenset(cfg.get('ignore', []))
  167. while files:
  168. f = files.popleft()
  169. if cfg.get('verbose'):
  170. print("Validating %s" % f)
  171. for c in fetch_checks(cfg):
  172. try:
  173. # http://legacy.python.org/dev/peps/pep-3155/
  174. check_name = c.__class__.__qualname__
  175. except AttributeError:
  176. check_name = ".".join([c.__class__.__module__,
  177. c.__class__.__name__])
  178. error_counts.setdefault(check_name, 0)
  179. try:
  180. extension_matcher = c.EXT_MATCHER
  181. except AttributeError:
  182. pass
  183. else:
  184. if not extension_matcher.match(f.extension):
  185. if cfg.get('verbose'):
  186. print(" Skipping check '%s' since it does not"
  187. " understand parsing a file with extension '%s'"
  188. % (check_name, f.extension))
  189. continue
  190. try:
  191. reports = set(c.REPORTS)
  192. except AttributeError:
  193. pass
  194. else:
  195. reports = reports - ignoreables
  196. if not reports:
  197. if cfg.get('verbose'):
  198. print(" Skipping check '%s', determined to only"
  199. " check ignoreable codes" % check_name)
  200. continue
  201. if cfg.get('verbose'):
  202. print(" Running check '%s'" % check_name)
  203. if isinstance(c, checks.ContentCheck):
  204. for line_num, code, message in c.report_iter(f):
  205. if code in ignoreables:
  206. continue
  207. if cfg.get('verbose'):
  208. print(' - %s:%s: %s %s'
  209. % (f.filename, line_num, code, message))
  210. else:
  211. print('%s:%s: %s %s'
  212. % (f.filename, line_num, code, message))
  213. error_counts[check_name] += 1
  214. elif isinstance(c, checks.LineCheck):
  215. for line_num, line in enumerate(f.lines_iter(), 1):
  216. for code, message in c.report_iter(line):
  217. if code in ignoreables:
  218. continue
  219. if cfg.get('verbose'):
  220. print(' - %s:%s: %s %s'
  221. % (f.filename, line_num, code, message))
  222. else:
  223. print('%s:%s: %s %s'
  224. % (f.filename, line_num, code, message))
  225. error_counts[check_name] += 1
  226. else:
  227. raise TypeError("Unknown check type: %s, %s"
  228. % (type(c), c))
  229. return error_counts
  230. def main():
  231. parser = argparse.ArgumentParser(
  232. prog='doc8',
  233. description=__doc__,
  234. formatter_class=argparse.RawDescriptionHelpFormatter)
  235. default_configs = ", ".join(CONFIG_FILENAMES)
  236. parser.add_argument("paths", metavar='path', type=str, nargs='*',
  237. help=("Path to scan for doc files"
  238. " (default: current directory)."),
  239. default=[os.getcwd()])
  240. parser.add_argument("--config", metavar='path', action="append",
  241. help="User config file location"
  242. " (default: %s)." % default_configs,
  243. default=[])
  244. parser.add_argument("--allow-long-titles", action="store_true",
  245. help="Allow long section titles (default: False).",
  246. default=False)
  247. parser.add_argument("--ignore", action="append", metavar="code",
  248. help="Ignore the given error code(s).",
  249. type=split_set_type,
  250. default=[])
  251. parser.add_argument("--no-sphinx", action="store_false",
  252. help="Do not ignore sphinx specific false positives.",
  253. default=True, dest='sphinx')
  254. parser.add_argument("--ignore-path", action="append", default=[],
  255. help="Ignore the given directory or file (globs"
  256. " are supported).", metavar='path')
  257. parser.add_argument("--default-extension", action="store",
  258. help="Default file extension to use when a file is"
  259. " found without a file extension.",
  260. default='', dest='default_extension',
  261. metavar='extension')
  262. parser.add_argument("--file-encoding", action="store",
  263. help="Override encoding to use when attempting"
  264. " to determine an input files text encoding "
  265. "(providing this avoids using `chardet` to"
  266. " automatically detect encoding/s)",
  267. default='', dest='file_encoding',
  268. metavar='encoding')
  269. parser.add_argument("--max-line-length", action="store", metavar="int",
  270. type=int,
  271. help="Maximum allowed line"
  272. " length (default: %s)." % MAX_LINE_LENGTH,
  273. default=MAX_LINE_LENGTH)
  274. parser.add_argument("-e", "--extension", action="append",
  275. metavar="extension",
  276. help="Check file extensions of the given type"
  277. " (default: %s)." % ", ".join(FILE_PATTERNS),
  278. default=list(FILE_PATTERNS))
  279. parser.add_argument("-v", "--verbose", dest="verbose", action='store_true',
  280. help="Run in verbose mode.", default=False)
  281. parser.add_argument("--version", dest="version", action='store_true',
  282. help="Show the version and exit.", default=False)
  283. args = vars(parser.parse_args())
  284. if args.get('version'):
  285. print(version.version_string())
  286. return 0
  287. args['ignore'] = merge_sets(args['ignore'])
  288. cfg = extract_config(args)
  289. args['ignore'].update(cfg.pop("ignore", set()))
  290. if 'sphinx' in cfg:
  291. args['sphinx'] = cfg.pop("sphinx")
  292. args['extension'].extend(cfg.pop('extension', []))
  293. args['ignore_path'].extend(cfg.pop('ignore_path', []))
  294. args.update(cfg)
  295. setup_logging(args.get('verbose'))
  296. files, files_ignored = scan(args)
  297. files_selected = len(files)
  298. error_counts = validate(args, files)
  299. total_errors = sum(six.itervalues(error_counts))
  300. print("=" * 8)
  301. print("Total files scanned = %s" % (files_selected))
  302. print("Total files ignored = %s" % (files_ignored))
  303. print("Total accumulated errors = %s" % (total_errors))
  304. if error_counts:
  305. print("Detailed error counts:")
  306. for check_name in sorted(six.iterkeys(error_counts)):
  307. check_errors = error_counts[check_name]
  308. print(" - %s = %s" % (check_name, check_errors))
  309. if total_errors:
  310. return 1
  311. else:
  312. return 0
  313. if __name__ == "__main__":
  314. sys.exit(main())