REST endpoint for searching Gerrit documentation.

The documentation of this endpoint is available in
rest-api-docsearch[.txt|.html].

The UI of showing the search result and doing the search will be in a
separate change.

Change-Id: Ifa4f5a7d576ada7f88a4fa1b765a38cba6d7e964

BUCK

@@ -3,8 +3,8 @@ include_defs('//tools/build.defs')
 gerrit_war(name = 'gerrit')
 gerrit_war(name = 'chrome',   ui = 'ui_chrome')
 gerrit_war(name = 'firefox',  ui = 'ui_firefox')
-gerrit_war(name = 'withdocs', context = DOCS)
+gerrit_war(name = 'withdocs', docs = True)
-gerrit_war(name = 'release',  context = DOCS + ['//plugins:core.zip'])
+gerrit_war(name = 'release',  docs = True, context = ['//plugins:core.zip'])
 
 API_DEPS = [
   ':extension-api',

Documentation/BUCK

@@ -3,7 +3,6 @@ include_defs('//Documentation/config.defs')
 include_defs('//tools/git.defs')
 
 DOC_DIR = 'Documentation'
-INDEX_DIR = DOC_DIR + '/.index'
 MAIN = ['//gerrit-pgm:pgm', '//gerrit-gwtui:ui_module']
 SRCS = glob(['*.txt'], excludes = ['licenses.txt'])
 
@@ -11,12 +10,10 @@ genrule(
   name = 'html',
   cmd = 'cd $TMP;' +
     'mkdir -p %s/images;' % DOC_DIR +
-    'unzip -q $SRCDIR/index.zip -d %s/;' % INDEX_DIR +
     'unzip -q $SRCDIR/only_html.zip -d %s/;' % DOC_DIR +
     'for s in $SRCS;do ln -s $s %s;done;' % DOC_DIR +
     'mv %s/*.{jpg,png} %s/images;' % (DOC_DIR, DOC_DIR) +
     'rm %s/only_html.zip;' % DOC_DIR +
-    'rm %s/index.zip;' % DOC_DIR +
     'rm %s/licenses.txt;' % DOC_DIR +
     'cp $SRCDIR/licenses.txt LICENSES.txt;' +
     'zip -qr $OUT *',
@@ -27,11 +24,9 @@ genrule(
       'doc.css',
       genfile('licenses.txt'),
       genfile('only_html.zip'),
-      genfile('index.zip'),
     ],
   deps = [
     ':generate_html',
-    ':index',
     ':licenses.txt',
   ],
   out = 'html.zip',
@@ -79,3 +74,18 @@ genrule(
   ],
   out = 'index.zip',
 )
+
+genrule(
+  name = 'index_jar',
+  cmd = 'jar cf $OUT -C $SRCDIR index.zip',
+  srcs = [genfile('index.zip')],
+  deps = [':index'],
+  out = 'index.jar',
+)
+
+prebuilt_jar(
+  name = 'index_lib',
+  binary_jar = genfile('index.jar'),
+  deps = [':index_jar'],
+  visibility = ['PUBLIC'],
+)

Documentation/rest-api-documentation.txt

@@ -0,0 +1,151 @@
+Gerrit Code Review - /Documentation/ REST API
+=============================================
+
+This page describes the documentation search related REST endpoints.
+Please also take note of the general information on the
+link:rest-api.html[REST API].
+
+Please note that this feature is only usable with documentation built-in.
+You'll need to
+`buck build :withdocs`
+or
+`buck build :release`
+to test this feature.
+
+[[documentation-endpoints]]
+Documentation Search Endpoints
+------------------------------
+
+[[search-documentation]]
+Search Documentation
+~~~~~~~~~~~~~~~~~~~~
+[verse]
+'GET /Documentation/'
+
+With `q` parameter, search our documentation index for the terms.
+
+A list of link:#doc-result[DocResult] entities is returned describing the
+results.
+
+.Request
+----
+  GET /Documentation/?q=test HTTP/1.0
+----
+
+.Response
+----
+  HTTP/1.1 200 OK
+  Content-Disposition: attachment
+  Content-Type: application/json; charset=UTF-8
+
+  )]}'
+  [
+    {
+      "title": "Gerrit Code Review - REST API Developers\u0027 Notes",
+      "url": "Documentation/dev-rest-api.html"
+    },
+    {
+      "title": "Gerrit Code Review - REST API",
+      "url": "Documentation/rest-api.html"
+    },
+    {
+      "title": "Gerrit Code Review - JavaScript API",
+      "url": "Documentation/js-api.html"
+    },
+    {
+      "title": "Gerrit Code Review - /plugins/ REST API",
+      "url": "Documentation/rest-api-plugins.html"
+    },
+    {
+      "title": "Gerrit Code Review - /config/ REST API",
+      "url": "Documentation/rest-api-config.html"
+    },
+    {
+      "title": "Gerrit Code Review for Git",
+      "url": "Documentation/index.html"
+    },
+    {
+      "title": "Gerrit Code Review - /access/ REST API",
+      "url": "Documentation/rest-api-access.html"
+    },
+    {
+      "title": "Gerrit Code Review - Plugin Development",
+      "url": "Documentation/dev-plugins.html"
+    },
+    {
+      "title": "Gerrit Code Review - Developer Setup",
+      "url": "Documentation/dev-readme.html"
+    },
+    {
+      "title": "Gerrit Code Review - Hooks",
+      "url": "Documentation/config-hooks.html"
+    },
+    {
+      "title": "Change Screen - Introduction",
+      "url": "Documentation/intro-change-screen.html"
+    },
+    {
+      "title": "Gerrit Code Review - /groups/ REST API",
+      "url": "Documentation/rest-api-groups.html"
+    },
+    {
+      "title": "Gerrit Code Review - /accounts/ REST API",
+      "url": "Documentation/rest-api-accounts.html"
+    },
+    {
+      "title": "Gerrit Code Review - /projects/ REST API",
+      "url": "Documentation/rest-api-documentation.html"
+    },
+    {
+      "title": "Gerrit Code Review - /projects/ REST API",
+      "url": "Documentation/rest-api-projects.html"
+    },
+    {
+      "title": "Gerrit Code Review - Prolog Submit Rules Cookbook",
+      "url": "Documentation/prolog-cookbook.html"
+    },
+    {
+      "title": "Gerrit Code Review - /changes/ REST API",
+      "url": "Documentation/rest-api-changes.html"
+    },
+    {
+      "title": "Gerrit Code Review - Configuration",
+      "url": "Documentation/config-gerrit.html"
+    },
+    {
+      "title": "Gerrit Code Review - Access Controls",
+      "url": "Documentation/access-control.html"
+    },
+    {
+      "title": "Gerrit Code Review - Licenses",
+      "url": "Documentation/licenses.html"
+    }
+  ]
+----
+
+.Query documentation
+****
+get::/Documentation/?q=keyword
+****
+
+
+[[json-entities]]
+JSON Entities
+-------------
+
+[[doc-result]]
+DocResult
+~~~~~~~~~
+The `DocResult` entity contains information about a document.
+
+[options="header",width="50%",cols="1,^2,4"]
+|=========================
+|Field Name  ||Description
+|`title`     ||The title of the document.
+|`url`       ||The URL of the document.
+|=========================
+
+
+GERRIT
+------
+Part of link:index.html[Gerrit Code Review]

Documentation/rest-api.txt

@@ -23,6 +23,8 @@ link:rest-api-plugins.html[/plugins/]::
   Plugin related REST endpoints
 link:rest-api-projects.html[/projects/]::
   Project related REST endpoints
+link:rest-api-documentation.html[/Documentation/]::
+  Documentation related REST endpoints
 
 Protocol Details
 ----------------

gerrit-httpd/BUCK

@@ -1,11 +1,24 @@
-SRCS = glob(['src/main/java/**/*.java'])
+CONSTANTS_SRC = [
+  'src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java',
+]
+SRCS = glob(
+  ['src/main/java/**/*.java'],
+  excludes = CONSTANTS_SRC,
+)
 RESOURCES = glob(['src/main/resources/**/*'])
 
+java_library2(
+  name = 'constants',
+  srcs = CONSTANTS_SRC,
+  visibility = ['PUBLIC'],
+)
+
 java_library2(
   name = 'httpd',
   srcs = SRCS,
   resources = RESOURCES,
   deps = [
+    ':constants',
     '//gerrit-antlr:query_exception',
     '//gerrit-common:server',
     '//gerrit-extension-api:api',
@@ -30,6 +43,9 @@ java_library2(
     '//lib/jgit:jgit',
     '//lib/jgit:jgit-servlet',
     '//lib/log:api',
+    '//lib/lucene:analyzers-common',
+    '//lib/lucene:core',
+    '//lib/lucene:query-parser',
   ],
   compile_deps = ['//lib:servlet-api-3_0'],
   visibility = ['PUBLIC'],

gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java

@@ -30,6 +30,7 @@ import com.google.gerrit.httpd.rpc.account.AccountsRestApiServlet;
 import com.google.gerrit.httpd.rpc.change.ChangesRestApiServlet;
 import com.google.gerrit.httpd.rpc.change.DeprecatedChangeQueryServlet;
 import com.google.gerrit.httpd.rpc.config.ConfigRestApiServlet;
+import com.google.gerrit.httpd.rpc.doc.QueryDocumentationFilter;
 import com.google.gerrit.httpd.rpc.group.GroupsRestApiServlet;
 import com.google.gerrit.httpd.rpc.project.ProjectsRestApiServlet;
 import com.google.gerrit.reviewdb.client.Change;
@@ -110,6 +111,8 @@ class UrlModule extends ServletModule {
     serveRegex("^/(?:a/)?groups/(.*)?$").with(GroupsRestApiServlet.class);
     serveRegex("^/(?:a/)?projects/(.*)?$").with(ProjectsRestApiServlet.class);
 
+    filter("/Documentation/").through(QueryDocumentationFilter.class);
+
     if (cfg.deprecatedQuery) {
       serve("/query").with(DeprecatedChangeQueryServlet.class);
     }

gerrit-httpd/src/main/java/com/google/gerrit/httpd/restapi/RestApiServlet.java

@@ -607,7 +607,7 @@ public class RestApiServlet extends HttpServlet {
     throw new InstantiationException("Cannot make " + type);
   }
 
-  private static void replyJson(@Nullable HttpServletRequest req,
+  public static void replyJson(@Nullable HttpServletRequest req,
       HttpServletResponse res,
       Multimap<String, String> config,
       Object result)

gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java

@@ -0,0 +1,24 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.rpc.doc;
+
+public class Constants {
+
+  public static final String DOC_FIELD = "doc";
+  public static final String TITLE_FIELD = "title";
+  public static final String URL_FIELD = "url";
+
+  private Constants() {}
+}

gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/QueryDocumentationFilter.java

@@ -0,0 +1,191 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.rpc.doc;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.gerrit.httpd.restapi.RestApiServlet;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+public class QueryDocumentationFilter implements Filter {
+  private static final Logger log =
+      LoggerFactory.getLogger(QueryDocumentationFilter.class);
+
+  private static final String INDEX_PATH = "index.zip";
+  private static final Version LUCENE_VERSION = Version.LUCENE_44;
+
+  private IndexSearcher searcher;
+  private QueryParser parser;
+
+  protected static class DocResult {
+    public String title;
+    public String url;
+    public String content;
+  }
+
+  @Inject
+  QueryDocumentationFilter() {
+  }
+
+  @Override
+  public void init(FilterConfig filterConfig) {
+    try {
+      Directory dir = readIndexDirectory();
+      if (dir == null) {
+        searcher = null;
+        parser = null;
+        return;
+      }
+      IndexReader reader = DirectoryReader.open(dir);
+      searcher = new IndexSearcher(reader);
+      StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
+      parser = new QueryParser(LUCENE_VERSION, Constants.DOC_FIELD, analyzer);
+    } catch (IOException e) {
+      log.error("Cannot initialize documentation full text index", e);
+      searcher = null;
+      parser = null;
+    }
+  }
+
+  @Override
+  public void destroy() {
+  }
+
+  @Override
+  public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+      throws IOException, ServletException {
+    HttpServletRequest req = (HttpServletRequest) request;
+    if ("GET".equals(req.getMethod())
+        && !Strings.isNullOrEmpty(req.getParameter("q"))) {
+      HttpServletResponse rsp = (HttpServletResponse) response;
+      try {
+        List<DocResult> result = doQuery(request.getParameter("q"));
+        Multimap<String, String> config = LinkedHashMultimap.create();
+        RestApiServlet.replyJson(req, rsp, config, result);
+      } catch (DocQueryException e) {
+        rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      }
+    } else {
+      chain.doFilter(request, response);
+    }
+  }
+
+  private List<DocResult> doQuery(String q) throws DocQueryException {
+    if (parser == null || searcher == null) {
+      throw new DocQueryException("Not initialized");
+    }
+    try {
+      Query query = parser.parse(q);
+      TopDocs results = searcher.search(query, Integer.MAX_VALUE);
+      ScoreDoc[] hits = results.scoreDocs;
+      int totalHits = results.totalHits;
+
+      List<DocResult> out = Lists.newArrayListWithCapacity(totalHits);
+      for (int i = 0; i < totalHits; i++) {
+        DocResult result = new DocResult();
+        Document doc = searcher.doc(hits[i].doc);
+        result.url = doc.get(Constants.URL_FIELD);
+        result.title = doc.get(Constants.TITLE_FIELD);
+        out.add(result);
+      }
+      return out;
+    } catch (IOException e) {
+      throw new DocQueryException(e);
+    } catch (ParseException e) {
+      throw new DocQueryException(e);
+    }
+  }
+
+  protected Directory readIndexDirectory() throws IOException {
+    Directory dir = new RAMDirectory();
+    byte[] buffer = new byte[4096];
+    InputStream index =
+        QueryDocumentationFilter.class.getClassLoader().getResourceAsStream(INDEX_PATH);
+    if (index == null) {
+      log.warn("No index available");
+      return null;
+    }
+    ZipInputStream zip = new ZipInputStream(index);
+    try {
+      ZipEntry entry;
+      while ((entry = zip.getNextEntry()) != null) {
+        IndexOutput out = dir.createOutput(entry.getName(), null);
+        int count;
+        while ((count = zip.read(buffer)) != -1) {
+          out.writeBytes(buffer, count);
+        }
+        out.close();
+      }
+    } finally {
+      zip.close();
+    }
+    // We must NOT call dir.close() here, as DirectoryReader.open() expects an opened directory.
+    return dir;
+  }
+
+  private static class DocQueryException extends Exception {
+    public DocQueryException() {
+    }
+
+    public DocQueryException(String msg) {
+      super(msg);
+    }
+
+    public DocQueryException(String msg, Throwable e) {
+      super(msg, e);
+    }
+
+    public DocQueryException(Throwable e) {
+      super(e);
+    }
+  }
+}

lib/asciidoctor/BUCK

@@ -31,6 +31,7 @@ java_library(
   srcs = ['java/DocIndexer.java'],
   deps = [
     ':asciidoc_lib',
+    '//gerrit-httpd:constants',
     '//lib:args4j',
     '//lib:guava',
     '//lib/lucene:analyzers-common',

lib/asciidoctor/java/DocIndexer.java

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 import com.google.common.io.Files;
+import com.google.gerrit.httpd.rpc.doc.Constants;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.util.CharArraySet;
@@ -25,6 +26,7 @@ import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.util.Version;
+
 import org.kohsuke.args4j.Argument;
 import org.kohsuke.args4j.CmdLineException;
 import org.kohsuke.args4j.CmdLineParser;
@@ -43,9 +45,6 @@ import java.util.zip.ZipOutputStream;
 
 public class DocIndexer {
   private static final Version LUCENE_VERSION = Version.LUCENE_44;
-  private static final String DOC_FIELD = "doc";
-  private static final String URL_FIELD = "url";
-  private static final String TITLE_FIELD = "title";
 
   @Option(name = "-z", usage = "output zip file")
   private String zipFile;
@@ -100,10 +99,10 @@ public class DocIndexer {
           inputFile, inExt, outExt);
       FileReader reader = new FileReader(file);
       Document doc = new Document();
-      doc.add(new TextField(DOC_FIELD, reader));
+      doc.add(new TextField(Constants.DOC_FIELD, reader));
       doc.add(new StringField(
-            URL_FIELD, prefix + outputFile, Field.Store.YES));
+            Constants.URL_FIELD, prefix + outputFile, Field.Store.YES));
-      doc.add(new TextField(TITLE_FIELD, title, Field.Store.YES));
+      doc.add(new TextField(Constants.TITLE_FIELD, title, Field.Store.YES));
       iwriter.addDocument(doc);
       reader.close();
     }

lib/lucene/BUCK

@@ -40,6 +40,14 @@ maven_jar(
   license = 'Apache2.0',
 )
 
+maven_jar(
+  name = 'query-parser',
+  id = 'org.apache.lucene:lucene-queryparser:4.4.0',
+  bin_sha1 = 'e2fca26d9c64f3aad7b8a3461dbab14782107a06',
+  src_sha1 = 'f23e42ab90b5b7eb888d394282eba65362e88606',
+  license = 'Apache2.0',
+)
+
 maven_jar(
   name = 'spellchecker',
   id = 'org.apache.lucene:lucene-spellchecker:3.6.2',

tools/build.defs

@@ -14,7 +14,12 @@
 
 # These definitions support building a runnable version of Gerrit.
 
-DOCS = ['//Documentation:html.zip']
+DOCS_SRC = genfile('Documentation/html.zip')
+DOCS_LIB = '//Documentation:index_lib'
+DOCS_DEP = [
+  '//Documentation:html',
+  '//Documentation:index_lib',
+]
 LIBS = [
   '//gerrit-war:log4j-config',
   '//gerrit-war:init',
@@ -36,7 +41,8 @@ def war(
     libs = [],
     pgmlibs = [],
     context = [],
-    visibility = []
+    visibility = [],
+    docs = False
     ):
   cmd = ['$(exe //tools:pack_war)', '-o', '$OUT', '--tmp', '$TMP']
   for l in libs:
@@ -46,6 +52,10 @@ def war(
 
   src = []
   dep = []
+  if docs:
+    src.append(DOCS_SRC)
+    dep.extend(DOCS_DEP)
+    cmd.extend(['--lib', DOCS_LIB])
   if context:
     root = get_base_path()
     if root:
@@ -56,6 +66,7 @@ def war(
         r = root + r[2:]
       r = r.replace(':', '/')
       src.append(genfile(r))
+  if src:
     cmd.append('$SRCS')
 
   genrule(
@@ -67,7 +78,7 @@ def war(
     visibility = visibility,
   )
 
-def gerrit_war(name, ui = 'ui_optdbg', context = []):
+def gerrit_war(name, ui = 'ui_optdbg', context = [], docs = False):
   war(
     name = name,
     libs = LIBS + ['//gerrit-war:version'],
@@ -77,4 +88,5 @@ def gerrit_war(name, ui = 'ui_optdbg', context = []):
       '//gerrit-war:webapp_assets.zip',
       '//gerrit-gwtui:' + ui + '.zip',
     ] + context,
+    docs = docs,
   )

tools/eclipse/BUCK

@@ -13,5 +13,6 @@ java_library(
     '//lib/asciidoctor:asciidoc_lib',
     '//lib/asciidoctor:doc_indexer_lib',
     '//lib/prolog:compiler_lib',
+    '//Documentation:index_lib',
   ] + scan_plugins(),
 )