Do all asciidoctor rendering in a single jvm

Previously for every html rendered, there's a separate jvm used, and buck's
parallel building will execute multiple jvm at once, burning the CPU and being
very slow.

Modified the asciidoctor-java-integrator CLI java wrapper interface to do that
all in a single jvm.

On a workstation tested, "buck build Documentation:html" took about 140s before,
and 60s after.

As a side-effect, we lose the build rules of the single html files (e.g.
"buck build Documentation:licenses.html")

Change-Id: Ifc70c63676b59571c6e240636752b7cba9270f04

Documentation/BUCK

@@ -4,48 +4,41 @@ include_defs('//tools/git.defs')
 
 MAIN = ['//gerrit-pgm:pgm', '//gerrit-gwtui:ui_module']
 SRCS = glob(['*.txt'], excludes = ['licenses.txt'])
-HTML = [txt[0:-4] + '.html' for txt in SRCS]
 
 genrule(
   name = 'html',
   cmd = 'cd $TMP;' +
     'mkdir -p Documentation/images;' +
+    'unzip -q $SRCDIR/only_html.zip -d Documentation/;' +
     'for s in $SRCS;do ln -s $s Documentation;done;' +
     'mv Documentation/*.{jpg,png} Documentation/images;' +
+    'rm Documentation/only_html.zip;' +
     'rm Documentation/licenses.txt;' +
     'cp $SRCDIR/licenses.txt LICENSES.txt;' +
     'zip -qr $OUT *',
-  srcs = [genfile(d) for d in HTML] +
+  srcs = [genfile('only_html.zip')] +
     glob([
       'images/*.jpg',
       'images/*.png',
     ]) + [
-    genfile('doc.css'),
+    'doc.css',
-    genfile('licenses.html'),
     genfile('licenses.txt'),
   ],
-  deps = [':' + d for d in HTML] + [
+  deps = [
-    ':licenses.html',
+    ':generate_html',
     ':licenses.txt',
-    ':doc.css',
   ],
   out = 'html.zip',
   visibility = ['PUBLIC'],
 )
 
-genrule(
-  name = 'doc.css',
-  cmd = 'ln -s $SRCDIR/doc.css $OUT',
-  srcs = ['doc.css'],
-  out = 'doc.css',
-)
-
 genasciidoc(
+  name = 'generate_html',
   srcs = SRCS + [genfile('licenses.txt')],
-  outs = HTML + ['licenses.html'],
+  deps = [':licenses.txt'],
-  deps = DOCUMENTATION_DEPS,
   attributes = documentation_attributes(git_describe()),
   backend = 'html5',
+  out = 'only_html.zip',
 )
 
 genrule(

Documentation/asciidoc.defs

@@ -13,25 +13,30 @@
 # limitations under the License.
 
 def genasciidoc(
+    name,
+    out,
     srcs = [],
-    outs = [],
+    deps = [],
-    deps = {},
     attributes = [],
     backend = None,
     visibility = []):
   EXPN = '.expn'
 
-  asciidoc = ['$(exe //lib/asciidoctor:asciidoc)']
+  asciidoc = [
+      '$(exe //lib/asciidoctor:asciidoc)',
+      '-z', '$OUT',
+      '--in-ext', '".txt%s"' % EXPN,
+      '--out-ext', '".html"',
+  ]
   if backend:
     asciidoc.extend(['-b', backend])
   for attribute in attributes:
     asciidoc.extend(['-a', attribute])
-  asciidoc.extend(['-o', '$OUT'])
+  asciidoc.append('$SRCS')
-
+  newsrcs = []
-  for p in zip(srcs, outs):
+  newdeps = deps + ['//lib/asciidoctor:asciidoc']
-    src, out = p
-    dep = deps.get(src) or []
 
+  for src in srcs:
     tx = []
     fn = src
     if fn.startswith('BUCKGEN:') :
@@ -48,14 +53,14 @@ def genasciidoc(
       deps = tx + [':replace_macros'],
       out = ex,
     )
-    genrule(
+    newdeps.append(':' + ex)
-      name = out,
+    newsrcs.append(genfile(ex))
-      cmd = ' '.join(asciidoc + ['$SRCDIR/' + ex]),
+
-      srcs = [genfile(ex)] + [genfile(n + EXPN) for n in dep],
+  genrule(
-      deps = [':' + n + EXPN for n in dep] + [
+    name = name,
-        ':' + ex,
+    cmd = ' '.join(asciidoc),
-        '//lib/asciidoctor:asciidoc',
+    srcs = newsrcs,
-      ],
+    deps = newdeps,
-      out = out,
+    out = out,
-      visibility = visibility,
+    visibility = visibility,
-    )
+  )

lib/asciidoctor/BUCK

@@ -2,11 +2,21 @@ include_defs('//lib/maven.defs')
 
 java_binary(
   name = 'asciidoc',
-  main_class = 'org.asciidoctor.cli.AsciidoctorInvoker',
+  main_class = 'Main',
-  deps = [':asciidoctor'],
+  deps = [':main_lib'],
   visibility = ['PUBLIC'],
 )
 
+java_library(
+  name = 'main_lib',
+  srcs = ['java/Main.java'],
+  deps = [
+    ':asciidoctor',
+    ':jruby',
+    '//lib:args4j',
+  ],
+)
+
 maven_jar(
   name = 'asciidoctor',
   id = 'org.asciidoctor:asciidoctor-java-integration:0.1.3',
@@ -14,19 +24,6 @@ maven_jar(
   license = 'Apache2.0',
   visibility = [],
   attach_source = False,
-  deps = [
-    ':jcommander',
-    ':jruby',
-  ],
-)
-
-maven_jar(
-  name = 'jcommander',
-  id = 'com.beust:jcommander:1.30',
-  sha1 = 'c440b30a944ba199751551aee393f8aa03b3c327',
-  license = 'Apache2.0',
-  visibility = [],
-  attach_source = False,
 )
 
 maven_jar(

lib/asciidoctor/java/Main.java

@@ -0,0 +1,159 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.asciidoctor.Asciidoctor;
+import org.asciidoctor.AttributesBuilder;
+import org.asciidoctor.Options;
+import org.asciidoctor.OptionsBuilder;
+import org.asciidoctor.internal.JRubyAsciidoctor;
+
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+public class Main {
+
+  private static final int BUFSIZ = 4096;
+  private static final String DOCTYPE = "article";
+  private static final String ERUBY = "erb";
+
+  @Option(name = "-b", usage = "set output format backend")
+  private String backend = "html5";
+
+  @Option(name = "-z", usage = "output zip file")
+  private String zipFile;
+
+  @Option(name = "--in-ext", usage = "extension for input files")
+  private String inExt = ".txt";
+
+  @Option(name = "--out-ext", usage = "extension for output files")
+  private String outExt = ".html";
+
+  @Option(name = "-a", usage =
+      "a list of attributes, in the form key or key=value pair")
+  private List<String> attributes = new ArrayList<String>();
+
+  @Argument(usage = "input files")
+  private List<String> inputFiles = new ArrayList<String>();
+
+  private String mapInFileToOutFile(String inFile) {
+    String basename = new File(inFile).getName();
+    if (basename.endsWith(inExt)) {
+      basename = basename.substring(0, basename.length() - inExt.length());
+    } else {
+      // Strip out the last extension
+      int pos = basename.lastIndexOf('.');
+      if (pos > 0) {
+        basename = basename.substring(0, pos);
+      }
+    }
+    return basename + outExt;
+  }
+
+  private Options createOptions(File tmpFile) {
+    OptionsBuilder optionsBuilder = OptionsBuilder.options();
+
+    optionsBuilder.backend(backend).docType(DOCTYPE).eruby(ERUBY);
+    // XXX(fishywang): ideally we should just output to a string and add the
+    // content into zip. But asciidoctor will actually ignore all attributes if
+    // not output to a file. So we *have* to output to a file then read the
+    // content of the file into zip.
+    optionsBuilder.toFile(tmpFile);
+
+    AttributesBuilder attributesBuilder = AttributesBuilder.attributes();
+    attributesBuilder.attributes(getAttributes());
+    optionsBuilder.attributes(attributesBuilder.get());
+
+    return optionsBuilder.get();
+  }
+
+  private Map<String, Object> getAttributes() {
+    Map<String, Object> attributeValues = new HashMap<String, Object>();
+
+    for (String attribute : attributes) {
+      int equalsIndex = attribute.indexOf('=');
+      if(equalsIndex > -1) {
+        String name = attribute.substring(0, equalsIndex);
+        String value = attribute.substring(equalsIndex + 1, attribute.length());
+
+        attributeValues.put(name, value);
+      } else {
+        attributeValues.put(attribute, "");
+      }
+    }
+
+    return attributeValues;
+  }
+
+  private void invoke(String... parameters) throws IOException {
+    CmdLineParser parser = new CmdLineParser(this);
+    try {
+      parser.parseArgument(parameters);
+      if (inputFiles.isEmpty()) {
+        throw new CmdLineException(parser,
+            "asciidoctor: FAILED: input file missing");
+      }
+    } catch (CmdLineException e) {
+      System.err.println(e.getMessage());
+      parser.printUsage(System.err);
+      System.exit(1);
+      return;
+    }
+
+    ZipOutputStream zip = new ZipOutputStream(new FileOutputStream(zipFile));
+    byte[] buf = new byte[BUFSIZ];
+    for (String inputFile : inputFiles) {
+      File tmp = File.createTempFile("doc", ".html");
+      Options options = createOptions(tmp);
+      renderInput(options, inputFile);
+
+      FileInputStream input = new FileInputStream(tmp);
+      int len;
+      zip.putNextEntry(new ZipEntry(mapInFileToOutFile(inputFile)));
+      while ((len = input.read(buf)) > 0) {
+        zip.write(buf, 0, len);
+      }
+      input.close();
+      tmp.delete();
+      zip.closeEntry();
+    }
+    zip.close();
+  }
+
+  private void renderInput(Options options, String inputFile) {
+    Asciidoctor asciidoctor = JRubyAsciidoctor.create();
+    asciidoctor.renderFile(new File(inputFile), options);
+  }
+
+  public static void main(String[] args) {
+    try {
+      new Main().invoke(args);
+    } catch (IOException e) {
+      System.err.println(e.getMessage());
+      System.exit(1);
+    }
+  }
+}