validate modified files and build affected books
At the moment the validation script simply validates all files and builds all books afterwards. This results in a very long runtime (appr. 30 minutes at the moment). With this patch only modified files are validated and only affected books are built. A book is affected if one XML file includes a modified file. If no book is directly affected all books are built. In this way, resources can be saved and validation is performed much faster. Change-Id: I17dcf119c0bafc4656c9f267f0d3f5ae884ea76e
This commit is contained in:
parent
d0abb8c290
commit
7c4b4bd4a3
116
validate.py
116
validate.py
|
@ -34,6 +34,28 @@ FILE_EXCEPTIONS = ['ha-guide-docinfo.xml','bk001-ch003-associate-general.xml']
|
|||
BOOK_EXCEPTIONS = []
|
||||
|
||||
|
||||
# NOTE(berendt): check_output as provided in Python 2.7.5 to make script
|
||||
# usable with Python < 2.7
|
||||
def check_output(*popenargs, **kwargs):
|
||||
"""Run command with arguments and return its output as a byte string.
|
||||
|
||||
If the exit code was non-zero it raises a CalledProcessError. The
|
||||
CalledProcessError object will have the return code in the returncode
|
||||
attribute and output in the output attribute.
|
||||
"""
|
||||
if 'stdout' in kwargs:
|
||||
raise ValueError('stdout argument not allowed, it will be overridden.')
|
||||
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
|
||||
output, unused_err = process.communicate()
|
||||
retcode = process.poll()
|
||||
if retcode:
|
||||
cmd = kwargs.get("args")
|
||||
if cmd is None:
|
||||
cmd = popenargs[0]
|
||||
raise CalledProcessError(retcode, cmd, output=output)
|
||||
return output
|
||||
|
||||
|
||||
def get_schema():
|
||||
"""Return the DocBook RELAX NG schema"""
|
||||
url = "http://www.oasis-open.org/docbook/xml/5.0b5/rng/docbookxi.rng"
|
||||
|
@ -112,10 +134,27 @@ def error_message(error_log):
|
|||
return "\n".join(errs)
|
||||
|
||||
|
||||
def get_modified_files():
|
||||
try:
|
||||
args = ["git", "diff", "--name-only", "HEAD", "HEAD~1"]
|
||||
modified_files = check_output(args).strip().split()
|
||||
except (CalledProcessError, OSError) as e:
|
||||
print("git failed: %s" % e)
|
||||
sys.exit(1)
|
||||
|
||||
modified_files = map(lambda x: os.path.abspath(x), modified_files)
|
||||
return modified_files
|
||||
|
||||
|
||||
def validate_individual_files(rootdir, exceptions):
|
||||
schema = get_schema()
|
||||
|
||||
any_failures = False
|
||||
modified_files = get_modified_files()
|
||||
print("\nFollowing modified files were found:")
|
||||
for f in modified_files:
|
||||
print(">>> %s" % f)
|
||||
print("")
|
||||
|
||||
for root, dirs, files in os.walk(rootdir):
|
||||
# Don't descend into 'target' subdirectories
|
||||
|
@ -127,44 +166,86 @@ def validate_individual_files(rootdir, exceptions):
|
|||
|
||||
for f in files:
|
||||
# Ignore maven files, which are called pom.xml
|
||||
if f.endswith('.xml') and f != 'pom.xml' \
|
||||
and f not in exceptions:
|
||||
if (f.endswith('.xml') and
|
||||
f != 'pom.xml' and
|
||||
f not in exceptions):
|
||||
try:
|
||||
path = os.path.abspath(os.path.join(root, f))
|
||||
if path not in modified_files:
|
||||
print((">>> %s not modified. "
|
||||
"Skipping validation." % path))
|
||||
continue
|
||||
doc = etree.parse(path)
|
||||
if validation_failed(schema, doc):
|
||||
any_failures = True
|
||||
print error_message(schema.error_log)
|
||||
print(error_message(schema.error_log))
|
||||
verify_section_tags_have_xmid(doc)
|
||||
verify_nice_usage_of_whitespaces(os.path.join(root, f))
|
||||
except etree.XMLSyntaxError as e:
|
||||
any_failures = True
|
||||
print "%s: %s" % (path, e)
|
||||
print("%s: %s" % (path, e))
|
||||
except ValueError as e:
|
||||
any_failures = True
|
||||
print "%s: %s" % (path, e)
|
||||
print("%s: %s" % (path, e))
|
||||
|
||||
if any_failures:
|
||||
sys.exit(1)
|
||||
|
||||
def build_all_books(rootdir, exceptions):
|
||||
""" Build all of the books.
|
||||
|
||||
Looks for all directories with "pom.xml" in them and runs
|
||||
def build_affected_books(rootdir, book_exceptions, file_exceptions):
|
||||
"""Build all the books which are affected by modified files.
|
||||
|
||||
Looks for all directories with "pom.xml" and checks if a
|
||||
XML file in the directory includes a modified file. If at least
|
||||
one XML file includes a modified file the method calls
|
||||
"mvn clean generate-sources" in that directory.
|
||||
|
||||
This will throw an exception if a book fails to build
|
||||
"""
|
||||
modified_files = get_modified_files()
|
||||
print("\nFollowing modified files were found:")
|
||||
for f in modified_files:
|
||||
print(">>> %s" % f)
|
||||
print("")
|
||||
|
||||
affected_books = []
|
||||
books = []
|
||||
for root, dirs, files in os.walk(rootdir):
|
||||
book = os.path.basename(root)
|
||||
if ("pom.xml" in files) and (book not in exceptions):
|
||||
print "Building %s" % book
|
||||
if ("pom.xml" in files and
|
||||
os.path.basename(root) not in book_exceptions):
|
||||
books.append(root)
|
||||
os.chdir(root)
|
||||
subprocess.check_call(["mvn", "clean", "generate-sources"])
|
||||
for f in files:
|
||||
if (f.endswith('.xml') and
|
||||
f != 'pom.xml' and
|
||||
f not in file_exceptions):
|
||||
path = os.path.abspath(os.path.join(root, f))
|
||||
doc = etree.parse(path)
|
||||
ns = {"xi": "http://www.w3.org/2001/XInclude"}
|
||||
for node in doc.xpath('//xi:include', namespaces=ns):
|
||||
href = node.get('href')
|
||||
if (href.endswith('.xml') and
|
||||
f not in file_exceptions and
|
||||
os.path.abspath(href) in modified_files):
|
||||
affected_books.append(root)
|
||||
break
|
||||
if os.path.basename(root) in affected_books:
|
||||
break
|
||||
|
||||
if affected_books:
|
||||
books = affected_books
|
||||
else:
|
||||
print("No books are affected by modified files. Building all books.")
|
||||
|
||||
for book in books:
|
||||
print("Building %s\n" % os.path.basename(book))
|
||||
os.chdir(book)
|
||||
subprocess.check_call(["mvn", "clean", "generate-sources"])
|
||||
|
||||
|
||||
def main(rootdir):
|
||||
validate_individual_files(rootdir, FILE_EXCEPTIONS)
|
||||
build_all_books(rootdir, BOOK_EXCEPTIONS)
|
||||
build_affected_books(rootdir, BOOK_EXCEPTIONS, FILE_EXCEPTIONS)
|
||||
|
||||
|
||||
def default_root():
|
||||
|
@ -172,8 +253,13 @@ def default_root():
|
|||
|
||||
The current working directory must be inside of the openstack-manuals
|
||||
repository for this method to succeed"""
|
||||
args = ["git", "rev-parse", "--show-toplevel"]
|
||||
gitroot = subprocess.check_output(args).rstrip()
|
||||
try:
|
||||
args = ["git", "rev-parse", "--show-toplevel"]
|
||||
gitroot = check_output(args).rstrip()
|
||||
except (CalledProcessError, OSError) as e:
|
||||
print("git failed: %s" % e)
|
||||
sys.exit(1)
|
||||
|
||||
return os.path.join(gitroot, "doc/src/docbkx")
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
Loading…
Reference in New Issue