diff --git a/BUCK b/BUCK index 616a0fe06a..3d1541c379 100644 --- a/BUCK +++ b/BUCK @@ -3,8 +3,8 @@ include_defs('//tools/build.defs') gerrit_war(name = 'gerrit') gerrit_war(name = 'chrome', ui = 'ui_chrome') gerrit_war(name = 'firefox', ui = 'ui_firefox') -gerrit_war(name = 'withdocs', context = DOCS) -gerrit_war(name = 'release', context = DOCS + ['//plugins:core.zip']) +gerrit_war(name = 'withdocs', docs = True) +gerrit_war(name = 'release', docs = True, context = ['//plugins:core.zip']) API_DEPS = [ ':extension-api', diff --git a/Documentation/BUCK b/Documentation/BUCK index 71d8664ddb..b2b7d2ab6b 100644 --- a/Documentation/BUCK +++ b/Documentation/BUCK @@ -3,7 +3,6 @@ include_defs('//Documentation/config.defs') include_defs('//tools/git.defs') DOC_DIR = 'Documentation' -INDEX_DIR = DOC_DIR + '/.index' MAIN = ['//gerrit-pgm:pgm', '//gerrit-gwtui:ui_module'] SRCS = glob(['*.txt'], excludes = ['licenses.txt']) @@ -11,12 +10,10 @@ genrule( name = 'html', cmd = 'cd $TMP;' + 'mkdir -p %s/images;' % DOC_DIR + - 'unzip -q $SRCDIR/index.zip -d %s/;' % INDEX_DIR + 'unzip -q $SRCDIR/only_html.zip -d %s/;' % DOC_DIR + 'for s in $SRCS;do ln -s $s %s;done;' % DOC_DIR + 'mv %s/*.{jpg,png} %s/images;' % (DOC_DIR, DOC_DIR) + 'rm %s/only_html.zip;' % DOC_DIR + - 'rm %s/index.zip;' % DOC_DIR + 'rm %s/licenses.txt;' % DOC_DIR + 'cp $SRCDIR/licenses.txt LICENSES.txt;' + 'zip -qr $OUT *', @@ -27,11 +24,9 @@ genrule( 'doc.css', genfile('licenses.txt'), genfile('only_html.zip'), - genfile('index.zip'), ], deps = [ ':generate_html', - ':index', ':licenses.txt', ], out = 'html.zip', @@ -79,3 +74,18 @@ genrule( ], out = 'index.zip', ) + +genrule( + name = 'index_jar', + cmd = 'jar cf $OUT -C $SRCDIR index.zip', + srcs = [genfile('index.zip')], + deps = [':index'], + out = 'index.jar', +) + +prebuilt_jar( + name = 'index_lib', + binary_jar = genfile('index.jar'), + deps = [':index_jar'], + visibility = ['PUBLIC'], +) diff --git a/Documentation/rest-api-documentation.txt b/Documentation/rest-api-documentation.txt new file mode 100644 index 0000000000..db7dc35f86 --- /dev/null +++ b/Documentation/rest-api-documentation.txt @@ -0,0 +1,151 @@ +Gerrit Code Review - /Documentation/ REST API +============================================= + +This page describes the documentation search related REST endpoints. +Please also take note of the general information on the +link:rest-api.html[REST API]. + +Please note that this feature is only usable with documentation built-in. +You'll need to +`buck build :withdocs` +or +`buck build :release` +to test this feature. + +[[documentation-endpoints]] +Documentation Search Endpoints +------------------------------ + +[[search-documentation]] +Search Documentation +~~~~~~~~~~~~~~~~~~~~ +[verse] +'GET /Documentation/' + +With `q` parameter, search our documentation index for the terms. + +A list of link:#doc-result[DocResult] entities is returned describing the +results. + +.Request +---- + GET /Documentation/?q=test HTTP/1.0 +---- + +.Response +---- + HTTP/1.1 200 OK + Content-Disposition: attachment + Content-Type: application/json; charset=UTF-8 + + )]}' + [ + { + "title": "Gerrit Code Review - REST API Developers\u0027 Notes", + "url": "Documentation/dev-rest-api.html" + }, + { + "title": "Gerrit Code Review - REST API", + "url": "Documentation/rest-api.html" + }, + { + "title": "Gerrit Code Review - JavaScript API", + "url": "Documentation/js-api.html" + }, + { + "title": "Gerrit Code Review - /plugins/ REST API", + "url": "Documentation/rest-api-plugins.html" + }, + { + "title": "Gerrit Code Review - /config/ REST API", + "url": "Documentation/rest-api-config.html" + }, + { + "title": "Gerrit Code Review for Git", + "url": "Documentation/index.html" + }, + { + "title": "Gerrit Code Review - /access/ REST API", + "url": "Documentation/rest-api-access.html" + }, + { + "title": "Gerrit Code Review - Plugin Development", + "url": "Documentation/dev-plugins.html" + }, + { + "title": "Gerrit Code Review - Developer Setup", + "url": "Documentation/dev-readme.html" + }, + { + "title": "Gerrit Code Review - Hooks", + "url": "Documentation/config-hooks.html" + }, + { + "title": "Change Screen - Introduction", + "url": "Documentation/intro-change-screen.html" + }, + { + "title": "Gerrit Code Review - /groups/ REST API", + "url": "Documentation/rest-api-groups.html" + }, + { + "title": "Gerrit Code Review - /accounts/ REST API", + "url": "Documentation/rest-api-accounts.html" + }, + { + "title": "Gerrit Code Review - /projects/ REST API", + "url": "Documentation/rest-api-documentation.html" + }, + { + "title": "Gerrit Code Review - /projects/ REST API", + "url": "Documentation/rest-api-projects.html" + }, + { + "title": "Gerrit Code Review - Prolog Submit Rules Cookbook", + "url": "Documentation/prolog-cookbook.html" + }, + { + "title": "Gerrit Code Review - /changes/ REST API", + "url": "Documentation/rest-api-changes.html" + }, + { + "title": "Gerrit Code Review - Configuration", + "url": "Documentation/config-gerrit.html" + }, + { + "title": "Gerrit Code Review - Access Controls", + "url": "Documentation/access-control.html" + }, + { + "title": "Gerrit Code Review - Licenses", + "url": "Documentation/licenses.html" + } + ] +---- + +.Query documentation +**** +get::/Documentation/?q=keyword +**** + + +[[json-entities]] +JSON Entities +------------- + +[[doc-result]] +DocResult +~~~~~~~~~ +The `DocResult` entity contains information about a document. + +[options="header",width="50%",cols="1,^2,4"] +|========================= +|Field Name ||Description +|`title` ||The title of the document. +|`url` ||The URL of the document. +|========================= + + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] diff --git a/Documentation/rest-api.txt b/Documentation/rest-api.txt index 7eed6ef363..788f2224ba 100644 --- a/Documentation/rest-api.txt +++ b/Documentation/rest-api.txt @@ -23,6 +23,8 @@ link:rest-api-plugins.html[/plugins/]:: Plugin related REST endpoints link:rest-api-projects.html[/projects/]:: Project related REST endpoints +link:rest-api-documentation.html[/Documentation/]:: + Documentation related REST endpoints Protocol Details ---------------- diff --git a/gerrit-httpd/BUCK b/gerrit-httpd/BUCK index 512e6e6554..3ec001c9c1 100644 --- a/gerrit-httpd/BUCK +++ b/gerrit-httpd/BUCK @@ -1,11 +1,24 @@ -SRCS = glob(['src/main/java/**/*.java']) +CONSTANTS_SRC = [ + 'src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java', +] +SRCS = glob( + ['src/main/java/**/*.java'], + excludes = CONSTANTS_SRC, +) RESOURCES = glob(['src/main/resources/**/*']) +java_library2( + name = 'constants', + srcs = CONSTANTS_SRC, + visibility = ['PUBLIC'], +) + java_library2( name = 'httpd', srcs = SRCS, resources = RESOURCES, deps = [ + ':constants', '//gerrit-antlr:query_exception', '//gerrit-common:server', '//gerrit-extension-api:api', @@ -30,6 +43,9 @@ java_library2( '//lib/jgit:jgit', '//lib/jgit:jgit-servlet', '//lib/log:api', + '//lib/lucene:analyzers-common', + '//lib/lucene:core', + '//lib/lucene:query-parser', ], compile_deps = ['//lib:servlet-api-3_0'], visibility = ['PUBLIC'], diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java index bf39bfb893..3c4dfc5d35 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java @@ -30,6 +30,7 @@ import com.google.gerrit.httpd.rpc.account.AccountsRestApiServlet; import com.google.gerrit.httpd.rpc.change.ChangesRestApiServlet; import com.google.gerrit.httpd.rpc.change.DeprecatedChangeQueryServlet; import com.google.gerrit.httpd.rpc.config.ConfigRestApiServlet; +import com.google.gerrit.httpd.rpc.doc.QueryDocumentationFilter; import com.google.gerrit.httpd.rpc.group.GroupsRestApiServlet; import com.google.gerrit.httpd.rpc.project.ProjectsRestApiServlet; import com.google.gerrit.reviewdb.client.Change; @@ -110,6 +111,8 @@ class UrlModule extends ServletModule { serveRegex("^/(?:a/)?groups/(.*)?$").with(GroupsRestApiServlet.class); serveRegex("^/(?:a/)?projects/(.*)?$").with(ProjectsRestApiServlet.class); + filter("/Documentation/").through(QueryDocumentationFilter.class); + if (cfg.deprecatedQuery) { serve("/query").with(DeprecatedChangeQueryServlet.class); } diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/restapi/RestApiServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/restapi/RestApiServlet.java index 517b0174c6..2e1cc0663d 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/restapi/RestApiServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/restapi/RestApiServlet.java @@ -607,7 +607,7 @@ public class RestApiServlet extends HttpServlet { throw new InstantiationException("Cannot make " + type); } - private static void replyJson(@Nullable HttpServletRequest req, + public static void replyJson(@Nullable HttpServletRequest req, HttpServletResponse res, Multimap config, Object result) diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java new file mode 100644 index 0000000000..886e9c5678 --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java @@ -0,0 +1,24 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.rpc.doc; + +public class Constants { + + public static final String DOC_FIELD = "doc"; + public static final String TITLE_FIELD = "title"; + public static final String URL_FIELD = "url"; + + private Constants() {} +} diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/QueryDocumentationFilter.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/QueryDocumentationFilter.java new file mode 100644 index 0000000000..50dd00fe73 --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/QueryDocumentationFilter.java @@ -0,0 +1,191 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.rpc.doc; + +import com.google.common.base.Strings; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.gerrit.httpd.restapi.RestApiServlet; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@Singleton +public class QueryDocumentationFilter implements Filter { + private static final Logger log = + LoggerFactory.getLogger(QueryDocumentationFilter.class); + + private static final String INDEX_PATH = "index.zip"; + private static final Version LUCENE_VERSION = Version.LUCENE_44; + + private IndexSearcher searcher; + private QueryParser parser; + + protected static class DocResult { + public String title; + public String url; + public String content; + } + + @Inject + QueryDocumentationFilter() { + } + + @Override + public void init(FilterConfig filterConfig) { + try { + Directory dir = readIndexDirectory(); + if (dir == null) { + searcher = null; + parser = null; + return; + } + IndexReader reader = DirectoryReader.open(dir); + searcher = new IndexSearcher(reader); + StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION); + parser = new QueryParser(LUCENE_VERSION, Constants.DOC_FIELD, analyzer); + } catch (IOException e) { + log.error("Cannot initialize documentation full text index", e); + searcher = null; + parser = null; + } + } + + @Override + public void destroy() { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest req = (HttpServletRequest) request; + if ("GET".equals(req.getMethod()) + && !Strings.isNullOrEmpty(req.getParameter("q"))) { + HttpServletResponse rsp = (HttpServletResponse) response; + try { + List result = doQuery(request.getParameter("q")); + Multimap config = LinkedHashMultimap.create(); + RestApiServlet.replyJson(req, rsp, config, result); + } catch (DocQueryException e) { + rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } else { + chain.doFilter(request, response); + } + } + + private List doQuery(String q) throws DocQueryException { + if (parser == null || searcher == null) { + throw new DocQueryException("Not initialized"); + } + try { + Query query = parser.parse(q); + TopDocs results = searcher.search(query, Integer.MAX_VALUE); + ScoreDoc[] hits = results.scoreDocs; + int totalHits = results.totalHits; + + List out = Lists.newArrayListWithCapacity(totalHits); + for (int i = 0; i < totalHits; i++) { + DocResult result = new DocResult(); + Document doc = searcher.doc(hits[i].doc); + result.url = doc.get(Constants.URL_FIELD); + result.title = doc.get(Constants.TITLE_FIELD); + out.add(result); + } + return out; + } catch (IOException e) { + throw new DocQueryException(e); + } catch (ParseException e) { + throw new DocQueryException(e); + } + } + + protected Directory readIndexDirectory() throws IOException { + Directory dir = new RAMDirectory(); + byte[] buffer = new byte[4096]; + InputStream index = + QueryDocumentationFilter.class.getClassLoader().getResourceAsStream(INDEX_PATH); + if (index == null) { + log.warn("No index available"); + return null; + } + ZipInputStream zip = new ZipInputStream(index); + try { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + IndexOutput out = dir.createOutput(entry.getName(), null); + int count; + while ((count = zip.read(buffer)) != -1) { + out.writeBytes(buffer, count); + } + out.close(); + } + } finally { + zip.close(); + } + // We must NOT call dir.close() here, as DirectoryReader.open() expects an opened directory. + return dir; + } + + private static class DocQueryException extends Exception { + public DocQueryException() { + } + + public DocQueryException(String msg) { + super(msg); + } + + public DocQueryException(String msg, Throwable e) { + super(msg, e); + } + + public DocQueryException(Throwable e) { + super(e); + } + } +} diff --git a/lib/asciidoctor/BUCK b/lib/asciidoctor/BUCK index 0b07b694ef..344710384d 100644 --- a/lib/asciidoctor/BUCK +++ b/lib/asciidoctor/BUCK @@ -31,6 +31,7 @@ java_library( srcs = ['java/DocIndexer.java'], deps = [ ':asciidoc_lib', + '//gerrit-httpd:constants', '//lib:args4j', '//lib:guava', '//lib/lucene:analyzers-common', diff --git a/lib/asciidoctor/java/DocIndexer.java b/lib/asciidoctor/java/DocIndexer.java index 497cba5807..2d04a6637f 100644 --- a/lib/asciidoctor/java/DocIndexer.java +++ b/lib/asciidoctor/java/DocIndexer.java @@ -13,6 +13,7 @@ // limitations under the License. import com.google.common.io.Files; +import com.google.gerrit.httpd.rpc.doc.Constants; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.util.CharArraySet; @@ -25,6 +26,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.Version; + import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; @@ -43,9 +45,6 @@ import java.util.zip.ZipOutputStream; public class DocIndexer { private static final Version LUCENE_VERSION = Version.LUCENE_44; - private static final String DOC_FIELD = "doc"; - private static final String URL_FIELD = "url"; - private static final String TITLE_FIELD = "title"; @Option(name = "-z", usage = "output zip file") private String zipFile; @@ -100,10 +99,10 @@ public class DocIndexer { inputFile, inExt, outExt); FileReader reader = new FileReader(file); Document doc = new Document(); - doc.add(new TextField(DOC_FIELD, reader)); + doc.add(new TextField(Constants.DOC_FIELD, reader)); doc.add(new StringField( - URL_FIELD, prefix + outputFile, Field.Store.YES)); - doc.add(new TextField(TITLE_FIELD, title, Field.Store.YES)); + Constants.URL_FIELD, prefix + outputFile, Field.Store.YES)); + doc.add(new TextField(Constants.TITLE_FIELD, title, Field.Store.YES)); iwriter.addDocument(doc); reader.close(); } diff --git a/lib/lucene/BUCK b/lib/lucene/BUCK index 38ece4c714..5973473191 100644 --- a/lib/lucene/BUCK +++ b/lib/lucene/BUCK @@ -40,6 +40,14 @@ maven_jar( license = 'Apache2.0', ) +maven_jar( + name = 'query-parser', + id = 'org.apache.lucene:lucene-queryparser:4.4.0', + bin_sha1 = 'e2fca26d9c64f3aad7b8a3461dbab14782107a06', + src_sha1 = 'f23e42ab90b5b7eb888d394282eba65362e88606', + license = 'Apache2.0', +) + maven_jar( name = 'spellchecker', id = 'org.apache.lucene:lucene-spellchecker:3.6.2', diff --git a/tools/build.defs b/tools/build.defs index b62c850451..4bb48ea035 100644 --- a/tools/build.defs +++ b/tools/build.defs @@ -14,7 +14,12 @@ # These definitions support building a runnable version of Gerrit. -DOCS = ['//Documentation:html.zip'] +DOCS_SRC = genfile('Documentation/html.zip') +DOCS_LIB = '//Documentation:index_lib' +DOCS_DEP = [ + '//Documentation:html', + '//Documentation:index_lib', +] LIBS = [ '//gerrit-war:log4j-config', '//gerrit-war:init', @@ -36,7 +41,8 @@ def war( libs = [], pgmlibs = [], context = [], - visibility = [] + visibility = [], + docs = False ): cmd = ['$(exe //tools:pack_war)', '-o', '$OUT', '--tmp', '$TMP'] for l in libs: @@ -46,6 +52,10 @@ def war( src = [] dep = [] + if docs: + src.append(DOCS_SRC) + dep.extend(DOCS_DEP) + cmd.extend(['--lib', DOCS_LIB]) if context: root = get_base_path() if root: @@ -56,6 +66,7 @@ def war( r = root + r[2:] r = r.replace(':', '/') src.append(genfile(r)) + if src: cmd.append('$SRCS') genrule( @@ -67,7 +78,7 @@ def war( visibility = visibility, ) -def gerrit_war(name, ui = 'ui_optdbg', context = []): +def gerrit_war(name, ui = 'ui_optdbg', context = [], docs = False): war( name = name, libs = LIBS + ['//gerrit-war:version'], @@ -77,4 +88,5 @@ def gerrit_war(name, ui = 'ui_optdbg', context = []): '//gerrit-war:webapp_assets.zip', '//gerrit-gwtui:' + ui + '.zip', ] + context, + docs = docs, ) diff --git a/tools/eclipse/BUCK b/tools/eclipse/BUCK index 9d6dd531b2..264e4ebdc0 100644 --- a/tools/eclipse/BUCK +++ b/tools/eclipse/BUCK @@ -13,5 +13,6 @@ java_library( '//lib/asciidoctor:asciidoc_lib', '//lib/asciidoctor:doc_indexer_lib', '//lib/prolog:compiler_lib', + '//Documentation:index_lib', ] + scan_plugins(), )