From 75ec972560d6ae5c82faaa002bf8780ae586ced4 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Mon, 23 Dec 2013 22:13:03 +0100 Subject: [PATCH] Initial support for gating api-site With these changes we can create new gates for checking the repository api-site as well as the API site repositories: doctest.py --force --check-build --api-site doctest.py --check-niceness --api-site doctest.py --check-syntax --api-site This checks also wadl files for syntax and niceness. Note that the other checks need further work. Also, --check-build really needs --force to build everything since the dependency generation does not handle the xml code of the repo yet. The building only works on api-site yet, needs more work for the other projects. The functions validate_all_files and validate_individual_files have been refactored to use one common function - thus removing some duplicated code. Change-Id: I214b4837401b2bcd46e5c6fa2e71b118254490d6 --- os_doc_tools/doctest.py | 195 +++++++++++++++++++++++++--------------- 1 file changed, 125 insertions(+), 70 deletions(-) diff --git a/os_doc_tools/doctest.py b/os_doc_tools/doctest.py index 3afd61e3..8251362f 100755 --- a/os_doc_tools/doctest.py +++ b/os_doc_tools/doctest.py @@ -76,13 +76,23 @@ def check_output(*popenargs, **kwargs): return output -def get_schema(): +def get_schema(is_api_site=False): """Return the DocBook RELAX NG schema""" - url = "http://docbook.org/xml/5.1CR1/rng/docbookxi.rng" + if is_api_site: + url = "http://docs.rackspace.com/rackbook/rackbook.rng" + else: + url = "http://docbook.org/xml/5.1CR1/rng/docbookxi.rng" relaxng_doc = etree.parse(urllib2.urlopen(url)) return etree.RelaxNG(relaxng_doc) +def get_wadl_schema(): + """Return the Wadl schema""" + url = "http://docs.rackspace.com/rackbook/wadl.xsd" + xmlschema_doc = etree.parse(urllib2.urlopen(url)) + return etree.XMLSchema(xmlschema_doc) + + def validation_failed(schema, doc): """Return True if the parsed doc fails against the schema @@ -393,63 +403,106 @@ def is_xml(filename): return filename.endswith('.xml') and not filename.endswith('/pom.xml') -def validate_individual_files(rootdir, exceptions, verbose, - check_syntax=False, check_niceness=False, - ignore_errors=False): - """Validate list of modified files.""" +def is_xml_wadl(filename): + """Returns true if file ends a valid .xml or .wadl file. + + Skips pom.xml files as well since those are not handled. + """ + + return (filename.endswith(('.xml', '.wadl')) and + not filename.endswith('/pom.xml')) + + +def is_wadl(filename): + """Returns true if file ends with .wadl""" + + return filename.endswith('.wadl') + + +def validate_individual_files(files_to_check, rootdir, exceptions, verbose, + check_syntax=False, check_niceness=False, + ignore_errors=False, is_api_site=False): + """Validate list of files.""" + + schema = get_schema(is_api_site) + if is_api_site: + wadl_schema = get_wadl_schema() - schema = get_schema() any_failures = False no_validated = 0 no_failed = 0 + if check_syntax and check_niceness: + print("Checking syntax and niceness of xml and wadl files...") + elif check_syntax: + print("Checking syntax of xml and wadl files...") + elif check_niceness: + print("Checking niceness of xml and wadl files...") + + for f in files_to_check: + base_f = os.path.basename(f) + if (base_f == "pom.xml" or + base_f in exceptions): + continue + # Files ending with ".xml" in subdirectories of + # wadls and samples files are not docbook files + # Currently we cannot validate these, so skip validation + # for them. + if (is_api_site and ("wadls" in f or "samples" in f)): + skip_xml_validation = True + else: + skip_xml_validation = False + + if (is_api_site and is_wadl(f)): + any_failures = validate_one_file(wadl_schema, rootdir, f, + verbose, + check_syntax, + check_niceness) + if any_failures: + no_failed = no_failed + 1 + no_validated = no_validated + 1 + elif (check_syntax and not skip_xml_validation) or check_niceness: + any_failures = validate_one_file(schema, rootdir, f, + verbose, + check_syntax and + not skip_xml_validation, + check_niceness) + if any_failures: + no_failed = no_failed + 1 + no_validated = no_validated + 1 + + if no_failed > 0: + print("Check failed, validated %d xml/wadl files with %d failures.\n" + % (no_validated, no_failed)) + if not ignore_errors: + sys.exit(1) + else: + print("Check passed, validated %d xml/wadl files.\n" % no_validated) + + +def validate_modified_files(rootdir, exceptions, verbose, + check_syntax=False, check_niceness=False, + ignore_errors=False, is_api_site=False): + """Validate list of modified files.""" + # Do not select deleted files, just Added, Copied, Modified, Renamed, # or Type changed modified_files = get_modified_files(rootdir, "--diff-filter=ACMRT") - modified_files = filter(is_xml, modified_files) - if check_syntax and check_niceness: - print("Checking syntax and niceness of xml files...") - elif check_syntax: - print("Checking syntax of xml files...") - elif check_niceness: - print("Checking niceness of xml files...") - modified_files = map(lambda x: os.path.abspath(x), modified_files) + modified_files = filter(is_xml_wadl, modified_files) - for f in modified_files: - base_f = os.path.basename(f) - if (base_f == "pom.xml" or - base_f in exceptions): - continue - any_failures = validate_one_file(schema, rootdir, f, verbose, - check_syntax, check_niceness) - if any_failures: - no_failed = no_failed + 1 - no_validated = no_validated + 1 - - if no_failed > 0: - print("Check failed, validated %d xml files with %d failures.\n" - % (no_validated, no_failed)) - if not ignore_errors: - sys.exit(1) - else: - print("Check passed, validated %d xml files.\n" % no_validated) + validate_individual_files(modified_files, rootdir, exceptions, + verbose, + check_syntax, check_niceness, + ignore_errors, is_api_site) def validate_all_files(rootdir, exceptions, verbose, check_syntax, check_niceness=False, - ignore_errors=False): + ignore_errors=False, is_api_site=False): """Validate all xml files.""" - schema = get_schema() - no_validated = 0 - no_failed = 0 - if check_syntax and check_niceness: - print("Checking syntax and niceness of all xml files...") - elif check_syntax: - print("Checking syntax of all xml files...") - elif check_niceness: - print("Checking niceness of all xml files...") + files_to_check = [] for root, dirs, files in os.walk(rootdir): # Don't descend into 'target' subdirectories @@ -461,24 +514,16 @@ def validate_all_files(rootdir, exceptions, verbose, for f in files: # Ignore maven files, which are called pom.xml - if (f.endswith('.xml') and + if (f.endswith(('.xml', '.wadl')) and f != 'pom.xml' and f not in exceptions): path = os.path.abspath(os.path.join(root, f)) - any_failures = validate_one_file( - schema, rootdir, path, verbose, - check_syntax, check_niceness) - if any_failures: - no_failed = no_failed + 1 - no_validated = no_validated + 1 + files_to_check.append(path) - if no_failed > 0: - print("Check failed, validated %d xml files with %d failures.\n" - % (no_validated, no_failed)) - if not ignore_errors: - sys.exit(1) - else: - print("Check passed, validated %d xml files.\n" % no_validated) + validate_individual_files(files_to_check, rootdir, exceptions, + verbose, + check_syntax, check_niceness, + ignore_errors, is_api_site) def logging_build_book(result): @@ -602,8 +647,8 @@ def find_affected_books(rootdir, book_exceptions, verbose, if os.path.basename(root) in book_exceptions: break - # Do not process files in doc itself - elif root.endswith('doc'): + # Do not process files in doc itself or top-level directory + elif root.endswith('doc') or root == rootdir: continue elif "pom.xml" in files: books.append(root) @@ -694,7 +739,8 @@ def find_affected_books(rootdir, book_exceptions, verbose, def build_affected_books(rootdir, book_exceptions, - verbose, force=False, ignore_errors=False): + verbose, force=False, ignore_errors=False, + is_api_site=False): """Build all the books which are affected by modified files. Looks for all directories with "pom.xml" and checks if a @@ -755,7 +801,7 @@ def main(): "the DocBook 5 RELAX NG schema") parser.add_argument('path', nargs='?', default=default_root(), help="Root directory that contains DocBook files, " - "defaults to `git rev-parse --show-toplevel`/doc") + "defaults to `git rev-parse --show-toplevel`") parser.add_argument("--force", help="Force the validation of all files " "and build all books", action="store_true") parser.add_argument("--check-build", help="Try to build books using " @@ -774,7 +820,13 @@ def main(): action="store_true") parser.add_argument("--verbose", help="Verbose execution", action="store_true") + parser.add_argument("--api-site", help="Special handling for " + "api-site repository", + action="store_true") + prog_args = parser.parse_args() + if not prog_args.api_site: + prog_args.path = os.path.join(prog_args.path, 'doc') if (len(sys.argv) == 1): # No arguments given, use check-all prog_args.check_all = True @@ -795,13 +847,15 @@ def main(): prog_args.verbose, prog_args.check_syntax, prog_args.check_niceness, - prog_args.ignore_errors) + prog_args.ignore_errors, + prog_args.api_site) else: - validate_individual_files(prog_args.path, FILE_EXCEPTIONS, - prog_args.verbose, - prog_args.check_syntax, - prog_args.check_niceness, - prog_args.ignore_errors) + validate_modified_files(prog_args.path, FILE_EXCEPTIONS, + prog_args.verbose, + prog_args.check_syntax, + prog_args.check_niceness, + prog_args.ignore_errors, + prog_args.api_site) if prog_args.check_deletions: check_deleted_files(prog_args.path, FILE_EXCEPTIONS, prog_args.verbose) @@ -809,11 +863,12 @@ def main(): if prog_args.check_build: build_affected_books(prog_args.path, BOOK_EXCEPTIONS, prog_args.verbose, prog_args.force, - prog_args.ignore_errors) + prog_args.ignore_errors, + prog_args.api_site) def default_root(): - """Return the location of openstack-manuals/doc/ + """Return the location of openstack-manuals The current working directory must be inside of the openstack-manuals repository for this method to succeed""" @@ -824,7 +879,7 @@ def default_root(): print("git failed: %s" % e) sys.exit(1) - return os.path.join(gitroot, "doc") + return gitroot if __name__ == "__main__": sys.exit(main())