Copy all generated books to common path

Copy all generated books to directory publish-docs in the git top-level
so that Jenkins can publish them on docs-draft.

Also copies www directory over if files in it where changed.

The tool now uses a config-file "doc-test-conf" which is expected in the
top-level of the repository.

blueprint draft-docs-on-docs-draft

Change-Id: Iea67050d827407f2f71585372c673f7f1dbccc46

README.rst

@@ -70,6 +70,10 @@ Release notes
 
  * Test that resources in wadl files have an xml:id (lp:bug 1275007)
  * Improve formatting of python command line clients (lp:bug 1274699)
+ * Copy all generated books to directory publish-docs in the git
+   top-level (lp:blueprint draft-docs-on-docs-draft)
+ * Requires now a config file in top-level git directory named
+   doc-test.conf
 
 0.4
 ---

doc-test.conf.sample

@@ -0,0 +1,17 @@
+[DEFAULT]
+
+repo_name = openstack-doc-tools
+api_site=True
+
+# From api-ref/src/wadls/object-api/src/
+file_exception=os-object-api-1.0.wadl
+
+ignore_dir=incubation
+ignore_dir=openstack-compute-api-1.0
+
+# These two options need to come as pairs:
+book=api-quick-start
+target_dir=target/docbkx/webhelp/api-quick-start-onepager-external
+
+book=api-ref
+target_dir=target/docbkx/html

doc/source/man/openstack-doc-test.rst

@@ -21,17 +21,9 @@ OPTIONS
 
   **General options**
 
-  **--verbose**
+  **--api-site**
-       Verbose execution.
+       Special handling for api-site and other API repositories
-
+       to handle WADL.
-  **--version**
-       Output version number.
-
-  **-, --help**
-        Show help message and exit.
-
-  **--force**
-        Force the validation of all files and build all books.
 
   **--check-build**
         Try to build books using modified files.
@@ -48,24 +40,37 @@ OPTIONS
   **--check-all**
        Run all checks (default if no arguments are given).
 
-  **--ignore-errors**
+  **--config-file PATH**
-       Do not exit on failures.
+       Path to a config file to use. Multiple config files can be
+       specified, with values in later files taking precedence.
 
-  **--api-site**
+  **--file-exception FILE_EXCEPTION**
-       Special handling for api-site and other API repositories
+      File that will be skipped during validation.
-       to handle WADL.
+
+  **--force**
+        Force the validation of all files and build all books.
+
+  **-h, --help**
+        Show help message and exit.
 
   **--ignore-dir IGNORE_DIR**
       Directory to ignore for building of manuals. The parameter can
       be passed multiple times to add several directories.
 
-  **--exceptions-file EXCEPTIONS_FILE**
+  **--ignore-errors**
-      File that contains filenames that will be skipped during validation.
+       Do not exit on failures.
+
+  **--verbose**
+       Verbose execution.
+
+  **--version**
+       Output version number.
 
 FILES
 =====
 
-None
+Reads the file `doc-test.conf` in the top-level directory of the git
+repository for option processing.
 
 SEE ALSO
 ========

os_doc_tools/doctest.py

@@ -26,13 +26,12 @@ Options:
 Ignores pom.xml files and subdirectories named "target".
 
 Requires:
-    - Python 2.7 or greater (for argparse)
+    - Python 2.7 or greater
     - lxml Python library
     - Maven
 
 '''
 
-import argparse
 import multiprocessing
 import os
 import re
@@ -43,6 +42,7 @@ import sys
 from lxml import etree
 
 import os_doc_tools
+from oslo.config import cfg
 
 
 # These are files that are known to not be in DocBook format.
@@ -52,6 +52,9 @@ FILE_EXCEPTIONS = []
 # These are books that we aren't checking yet
 BOOK_EXCEPTIONS = []
 
+# Mappings from books to directories
+BOOK_MAPPINGS = {}
+
 RESULTS_OF_BUILDS = []
 
 # List of recognized (allowable) os profiling directives.
@@ -263,8 +266,12 @@ def error_message(error_log):
     return "\n".join(errs)
 
 
-def only_www_touched():
+def www_touched(check_only_www):
-    """Check whether only files in www directory are touched."""
+    """Check whether files in www directory are touched.
+
+    If check_only_www is True: Check that only files in www are touched.
+    Otherwise check that files in www are touched.
+    """
 
     try:
         git_args = ["git", "diff", "--name-only", "HEAD~1", "HEAD"]
@@ -281,7 +288,9 @@ def only_www_touched():
         else:
             other_changed = True
 
-    return www_changed and not other_changed
+    if check_only_www:
+        return www_changed and not other_changed
+    return www_changed
 
 
 def ha_guide_touched():
@@ -599,7 +608,60 @@ def logging_build_book(result):
     RESULTS_OF_BUILDS.append(result)
 
 
-def build_book(book):
+def get_gitroot():
+    """Return path to top-level of git repository."""
+
+    try:
+        git_args = ["git", "rev-parse", "--show-toplevel"]
+        gitroot = check_output(git_args).rstrip()
+    except (subprocess.CalledProcessError, OSError) as e:
+        print("git failed: %s" % e)
+        sys.exit(1)
+
+    return gitroot
+
+
+def get_publish_path():
+    """Return path to use of publishing books."""
+
+    return os.path.join(get_gitroot(), 'publish-docs')
+
+
+def publish_www():
+    """Copy www files."""
+
+    publish_path = get_publish_path()
+    www_path = os.path.join(publish_path, 'www')
+    shutil.rmtree(www_path, ignore_errors=True)
+
+    source = os.path.join(get_gitroot(), 'www')
+    shutil.copytree(source, www_path)
+
+
+def publish_book(publish_path, book):
+    """Copy generated files to publish_path."""
+
+    # Assumption: The path for the book is the same as the name of directory
+    # the book is in. We need to special case any exceptions.
+
+    book_path = os.path.join(publish_path, book)
+
+    # Note that shutil.copytree does not allow an existing target directory,
+    # thus delete it.
+    shutil.rmtree(book_path, ignore_errors=True)
+
+    if os.path.isdir(os.path.join('target/docbkx/webhelp', book)):
+        source = os.path.join('target/docbkx/webhelp', book)
+    elif os.path.isdir(os.path.join('target/docbkx/webhelp/local', book)):
+        source = os.path.join('target/docbkx/webhelp/local', book)
+    elif (book in BOOK_MAPPINGS):
+        source = BOOK_MAPPINGS[book]
+
+    shutil.copytree(source, book_path,
+                    ignore=shutil.ignore_patterns('*.xml'))
+
+
+def build_book(book, publish_path):
     """Build book(s) in directory book."""
 
     # Note that we cannot build in parallel several books in the same
@@ -609,6 +671,7 @@ def build_book(book):
     result = True
     returncode = 0
     base_book = os.path.basename(book)
+    base_book_orig = base_book
     try:
         # Clean first and then build so that the output of all guides
         # is available
@@ -688,6 +751,7 @@ def build_book(book):
         returncode = e.returncode
         result = False
 
+    publish_book(publish_path, base_book_orig)
     return (base_book, result, output, returncode)
 
 
@@ -868,9 +932,10 @@ def build_affected_books(rootdir, book_exceptions, file_exceptions,
         maxjobs = 4
     pool = multiprocessing.Pool(maxjobs)
     print("Queuing the following books for building:")
+    publish_path = get_publish_path()
     for book in sorted(books):
         print("  %s" % os.path.basename(book))
-        pool.apply_async(build_book, (book, ),
+        pool.apply_async(build_book, (book, publish_path),
                          callback=logging_build_book)
     pool.close()
     print("Building all queued %d books now..." % len(books))
@@ -897,122 +962,145 @@ def build_affected_books(rootdir, book_exceptions, file_exceptions,
         print("Building of books finished successfully.\n")
 
 
-def parse_exceptions(exceptions_file, verbose):
+def add_exceptions(file_exception, verbose):
-    """Read list of exceptions from exceptions_file."""
+    """Add list of exceptions from file_exceptions."""
 
-    for line in open(exceptions_file, 'rU'):
+    for entry in file_exception:
-        if not line.startswith("#") and len(line) > 1:
+        if verbose:
-            filename = line.rstrip('\n\r')
+            print(" Adding file to ignore list: %s" % entry)
-            if verbose:
+        FILE_EXCEPTIONS.append(entry)
-                print("Adding file to ignore list: %s" % filename)
+
-            FILE_EXCEPTIONS.append(filename)
+
+cli_OPTS = [
+    cfg.BoolOpt("api-site", default=False,
+                help="Enable special handling for api-site repository."),
+    cfg.BoolOpt('check-all', default=False,
+                help="Run all checks."),
+    cfg.BoolOpt('check-build', default=False,
+                help="Check building of books using modified files."),
+    cfg.BoolOpt("check-deletions", default=False,
+                help="Check that deleted files are not used."),
+    cfg.BoolOpt("check-niceness", default=False,
+                help="Check the niceness of files, for example whitespace."),
+    cfg.BoolOpt("check-syntax", default=False,
+                help="Check the syntax of modified files"),
+    cfg.BoolOpt('force', default=False,
+                help="Force the validation of all files "
+                "and build all books."),
+    cfg.BoolOpt("ignore-errors", default=False,
+                help="Do not exit on failures."),
+    cfg.BoolOpt('verbose', default=False, short='v',
+                help="Verbose execution."),
+    cfg.StrOpt('exceptions-file',
+               help="Ignored, for compatibility only"),
+    cfg.MultiStrOpt("file-exception",
+                    help="File that will be skipped during validation."),
+    cfg.MultiStrOpt("ignore-dir",
+                    help="Directory to ignore for building of manuals. The "
+                         "parameter can be passed multiple times to add "
+                         "several directories."),
+]
+
+OPTS = [
+    # NOTE(jaegerandi): books and target-dirs could be a DictOpt
+    # but I could not get this working properly.
+    cfg.MultiStrOpt("book", default=None,
+                    help="Name of book that needs special mapping."),
+    cfg.MultiStrOpt("target-dir", default=None,
+                    help="Target directory for a book. The option "
+                    "must be in the same order as the book option."),
+    cfg.StrOpt("repo-name", default=None,
+               help="Name of repository."),
+]
+
+
+def handle_options():
+    """Setup configuration variables from config file and options."""
+
+    CONF = cfg.CONF
+    CONF.register_cli_opts(cli_OPTS)
+    CONF.register_opts(OPTS)
+
+    default_config_files = [os.path.join(get_gitroot(), 'doc-test.conf')]
+    CONF(sys.argv[1:], project='documentation', prog='openstack-doc-test',
+         version=os_doc_tools.__version__,
+         default_config_files=default_config_files)
+
+    if CONF.repo_name:
+        print ("Testing repository '%s'\n" % CONF.repo_name)
+
+    if CONF.verbose:
+        print("Verbose execution")
+    if CONF.file_exception:
+        add_exceptions(CONF.file_exception, CONF.verbose)
+
+    if (not CONF.check_build and not CONF.check_deletions and
+        not CONF.check_niceness and not CONF.check_syntax):
+        CONF.check_all = True
+
+    if CONF.check_all:
+        CONF.check_deletions = True
+        CONF.check_syntax = True
+        CONF.check_build = True
+        CONF.check_niceness = True
+
+    if CONF.check_build and CONF.book and CONF.target_dir:
+        if len(CONF.book) != len(CONF.target_dir):
+            print("ERROR: books and target-dirs need to have a 1:1 "
+                  "relationship.")
+            sys.exit(1)
+        for i in range(len(CONF.book)):
+            BOOK_MAPPINGS[CONF.book[i]] = CONF.target_dir[i]
+            if CONF.verbose:
+                print(" Target dir for %s is %s" %
+                      (CONF.book[i], BOOK_MAPPINGS[CONF.book[i]]))
 
 
 def main():
 
-    parser = argparse.ArgumentParser(description="Validate XML files against "
+    CONF = cfg.CONF
-                                     "the DocBook 5 RELAX NG schema")
+    print ("\nOpenStack Doc Checks (using openstack-doc-tools version %s)\n"
-    parser.add_argument('path', nargs='?', default=default_root(),
-                        help="Root directory that contains DocBook files, "
-                        "defaults to `git rev-parse --show-toplevel`")
-    parser.add_argument("--force", help="Force the validation of all files "
-                        "and build all books", action="store_true")
-    parser.add_argument("--check-build", help="Try to build books using "
-                        "modified files", action="store_true")
-    parser.add_argument("--check-syntax", help="Check the syntax of modified "
-                        "files", action="store_true")
-    parser.add_argument("--check-deletions", help="Check that deleted files "
-                        "are not used.", action="store_true")
-    parser.add_argument("--check-niceness", help="Check the niceness of "
-                        "files, for example whitespace.",
-                        action="store_true")
-    parser.add_argument("--check-all", help="Run all checks "
-                        "(default if no arguments are given)",
-                        action="store_true")
-    parser.add_argument("--ignore-errors", help="Do not exit on failures",
-                        action="store_true")
-    parser.add_argument("--verbose", help="Verbose execution",
-                        action="store_true")
-    parser.add_argument("--api-site", help="Special handling for "
-                        "api-site repository",
-                        action="store_true")
-    parser.add_argument("--ignore-dir",
-                        help="Directory to ignore for building of "
-                        "manuals. The parameter can be passed multiple "
-                        "times to add several directories.",
-                        action="append")
-    parser.add_argument("--exceptions-file",
-                        help="File that contains filenames that will "
-                        "be skipped during validation.")
-    parser.add_argument('--version',
-                        action='version',
-                        version=os_doc_tools.__version__)
-
-    prog_args = parser.parse_args()
-
-    print ("OpenStack Doc Checks (using openstack-doc-tools version %s)\n"
            % os_doc_tools.__version__)
-    if not prog_args.api_site:
-        prog_args.path = os.path.join(prog_args.path, 'doc')
-    if (len(sys.argv) == 1):
-        # No arguments given, use check-all
-        prog_args.check_all = True
 
-    if prog_args.exceptions_file:
+    handle_options()
-        parse_exceptions(prog_args.exceptions_file, prog_args.verbose)
 
-    if prog_args.check_all:
+    doc_path = get_gitroot()
-        prog_args.check_deletions = True
+    if not CONF.api_site:
-        prog_args.check_syntax = True
+        doc_path = os.path.join(doc_path, 'doc')
-        prog_args.check_build = True
-        prog_args.check_niceness = True
 
-    if not prog_args.force and only_www_touched():
+    if CONF.check_build and www_touched(False):
+        publish_www()
+
+    if not CONF.force and www_touched(True):
         print("Only files in www directory changed, nothing to do.\n")
         return
 
-    if prog_args.check_syntax or prog_args.check_niceness:
+    if CONF.check_syntax or CONF.check_niceness:
-        if prog_args.force:
+        if CONF.force:
-            validate_all_files(prog_args.path, FILE_EXCEPTIONS,
+            validate_all_files(doc_path, FILE_EXCEPTIONS,
-                               prog_args.verbose,
+                               CONF.verbose,
-                               prog_args.check_syntax,
+                               CONF.check_syntax,
-                               prog_args.check_niceness,
+                               CONF.check_niceness,
-                               prog_args.ignore_errors,
+                               CONF.ignore_errors,
-                               prog_args.api_site)
+                               CONF.api_site)
         else:
-            validate_modified_files(prog_args.path, FILE_EXCEPTIONS,
+            validate_modified_files(doc_path, FILE_EXCEPTIONS,
-                                    prog_args.verbose,
+                                    CONF.verbose,
-                                    prog_args.check_syntax,
+                                    CONF.check_syntax,
-                                    prog_args.check_niceness,
+                                    CONF.check_niceness,
-                                    prog_args.ignore_errors,
+                                    CONF.ignore_errors,
-                                    prog_args.api_site)
+                                    CONF.api_site)
 
-    if prog_args.check_deletions:
+    if CONF.check_deletions:
-        check_deleted_files(prog_args.path, FILE_EXCEPTIONS, prog_args.verbose)
+        check_deleted_files(doc_path, FILE_EXCEPTIONS, CONF.verbose)
 
-    if prog_args.check_build:
+    if CONF.check_build:
-        build_affected_books(prog_args.path, BOOK_EXCEPTIONS,
+        build_affected_books(doc_path, BOOK_EXCEPTIONS,
                              FILE_EXCEPTIONS,
-                             prog_args.verbose, prog_args.force,
+                             CONF.verbose, CONF.force,
-                             prog_args.ignore_errors,
+                             CONF.ignore_errors,
-                             prog_args.ignore_dir)
+                             CONF.ignore_dir)
 
 
-def default_root():
-    """Return the location of openstack-manuals
-
-    The current working directory must be inside of the openstack-manuals
-    repository for this method to succeed
-    """
-
-    try:
-        git_args = ["git", "rev-parse", "--show-toplevel"]
-        gitroot = check_output(git_args).rstrip()
-    except (subprocess.CalledProcessError, OSError) as e:
-        print("git failed: %s" % e)
-        sys.exit(1)
-
-    return gitroot
-
 if __name__ == "__main__":
     sys.exit(main())