opendev/gerrit
Code Issues Proposed changes

REST endpoint for searching Gerrit documentation.

Browse Source 
The documentation of this endpoint is available in
rest-api-docsearch[.txt|.html].

The UI of showing the search result and doing the search will be in a
separate change.

Change-Id: Ifa4f5a7d576ada7f88a4fa1b765a38cba6d7e964
This commit is contained in:
Yuxuan 'fishy' Wang 2013-10-14 12:39:30 -07:00
parent 6e4a19e9df
commit ec4b06a6e2
14 changed files with 436 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
BUCK
View File

@ -3,8 +3,8 @@ include_defs('//tools/build.defs')
gerrit_war(name = 'gerrit')
gerrit_war(name = 'chrome', ui = 'ui_chrome')
gerrit_war(name = 'firefox', ui = 'ui_firefox')
gerrit_war(name = 'withdocs', context = DOCS)
gerrit_war(name = 'release', context = DOCS + ['//plugins:core.zip'])
gerrit_war(name = 'withdocs', docs = True)
gerrit_war(name = 'release', docs = True, context = ['//plugins:core.zip'])
API_DEPS = [
':extension-api',

20
Documentation/BUCK
View File

@ -3,7 +3,6 @@ include_defs('//Documentation/config.defs')
include_defs('//tools/git.defs')
DOC_DIR = 'Documentation'
INDEX_DIR = DOC_DIR + '/.index'
MAIN = ['//gerrit-pgm:pgm', '//gerrit-gwtui:ui_module']
SRCS = glob(['*.txt'], excludes = ['licenses.txt'])
@ -11,12 +10,10 @@ genrule(
name = 'html',
cmd = 'cd $TMP;' +
'mkdir -p %s/images;' % DOC_DIR +
'unzip -q $SRCDIR/index.zip -d %s/;' % INDEX_DIR +
'unzip -q $SRCDIR/only_html.zip -d %s/;' % DOC_DIR +
'for s in $SRCS;do ln -s $s %s;done;' % DOC_DIR +
'mv %s/*.{jpg,png} %s/images;' % (DOC_DIR, DOC_DIR) +
'rm %s/only_html.zip;' % DOC_DIR +
'rm %s/index.zip;' % DOC_DIR +
'rm %s/licenses.txt;' % DOC_DIR +
'cp $SRCDIR/licenses.txt LICENSES.txt;' +
'zip -qr $OUT *',
@ -27,11 +24,9 @@ genrule(
'doc.css',
genfile('licenses.txt'),
genfile('only_html.zip'),
genfile('index.zip'),
],
deps = [
':generate_html',
':index',
':licenses.txt',
],
out = 'html.zip',
@ -79,3 +74,18 @@ genrule(
],
out = 'index.zip',
)
genrule(
name = 'index_jar',
cmd = 'jar cf $OUT -C $SRCDIR index.zip',
srcs = [genfile('index.zip')],
deps = [':index'],
out = 'index.jar',
)
prebuilt_jar(
name = 'index_lib',
binary_jar = genfile('index.jar'),
deps = [':index_jar'],
visibility = ['PUBLIC'],
)

151
Documentation/rest-api-documentation.txt Normal file
View File

@ -0,0 +1,151 @@
Gerrit Code Review - /Documentation/ REST API
=============================================
This page describes the documentation search related REST endpoints.
Please also take note of the general information on the
link:rest-api.html[REST API].
Please note that this feature is only usable with documentation built-in.
You'll need to
`buck build :withdocs`
or
`buck build :release`
to test this feature.
[[documentation-endpoints]]
Documentation Search Endpoints
------------------------------
[[search-documentation]]
Search Documentation
~~~~~~~~~~~~~~~~~~~~
[verse]
'GET /Documentation/'
With `q` parameter, search our documentation index for the terms.
A list of link:#doc-result[DocResult] entities is returned describing the
results.
.Request
----
GET /Documentation/?q=test HTTP/1.0
----
.Response
----
HTTP/1.1 200 OK
Content-Disposition: attachment
Content-Type: application/json; charset=UTF-8
)]}'
[
{
"title": "Gerrit Code Review - REST API Developers\u0027 Notes",
"url": "Documentation/dev-rest-api.html"
},
{
"title": "Gerrit Code Review - REST API",
"url": "Documentation/rest-api.html"
},
{
"title": "Gerrit Code Review - JavaScript API",
"url": "Documentation/js-api.html"
},
{
"title": "Gerrit Code Review - /plugins/ REST API",
"url": "Documentation/rest-api-plugins.html"
},
{
"title": "Gerrit Code Review - /config/ REST API",
"url": "Documentation/rest-api-config.html"
},
{
"title": "Gerrit Code Review for Git",
"url": "Documentation/index.html"
},
{
"title": "Gerrit Code Review - /access/ REST API",
"url": "Documentation/rest-api-access.html"
},
{
"title": "Gerrit Code Review - Plugin Development",
"url": "Documentation/dev-plugins.html"
},
{
"title": "Gerrit Code Review - Developer Setup",
"url": "Documentation/dev-readme.html"
},
{
"title": "Gerrit Code Review - Hooks",
"url": "Documentation/config-hooks.html"
},
{
"title": "Change Screen - Introduction",
"url": "Documentation/intro-change-screen.html"
},
{
"title": "Gerrit Code Review - /groups/ REST API",
"url": "Documentation/rest-api-groups.html"
},
{
"title": "Gerrit Code Review - /accounts/ REST API",
"url": "Documentation/rest-api-accounts.html"
},
{
"title": "Gerrit Code Review - /projects/ REST API",
"url": "Documentation/rest-api-documentation.html"
},
{
"title": "Gerrit Code Review - /projects/ REST API",
"url": "Documentation/rest-api-projects.html"
},
{
"title": "Gerrit Code Review - Prolog Submit Rules Cookbook",
"url": "Documentation/prolog-cookbook.html"
},
{
"title": "Gerrit Code Review - /changes/ REST API",
"url": "Documentation/rest-api-changes.html"
},
{
"title": "Gerrit Code Review - Configuration",
"url": "Documentation/config-gerrit.html"
},
{
"title": "Gerrit Code Review - Access Controls",
"url": "Documentation/access-control.html"
},
{
"title": "Gerrit Code Review - Licenses",
"url": "Documentation/licenses.html"
}
]
----
.Query documentation
****
get::/Documentation/?q=keyword
****
[[json-entities]]
JSON Entities
-------------
[[doc-result]]
DocResult
~~~~~~~~~
The `DocResult` entity contains information about a document.
[options="header",width="50%",cols="1,^2,4"]
|=========================
|Field Name ||Description
|`title` ||The title of the document.
|`url` ||The URL of the document.
|=========================
GERRIT
------
Part of link:index.html[Gerrit Code Review]

2
Documentation/rest-api.txt
View File

@ -23,6 +23,8 @@ link:rest-api-plugins.html[/plugins/]::
Plugin related REST endpoints
link:rest-api-projects.html[/projects/]::
Project related REST endpoints
link:rest-api-documentation.html[/Documentation/]::
Documentation related REST endpoints
Protocol Details
----------------

18
gerrit-httpd/BUCK
View File

@ -1,11 +1,24 @@
SRCS = glob(['src/main/java/**/*.java'])
CONSTANTS_SRC = [
'src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java',
]
SRCS = glob(
['src/main/java/**/*.java'],
excludes = CONSTANTS_SRC,
)
RESOURCES = glob(['src/main/resources/**/*'])
java_library2(
name = 'constants',
srcs = CONSTANTS_SRC,
visibility = ['PUBLIC'],
)
java_library2(
name = 'httpd',
srcs = SRCS,
resources = RESOURCES,
deps = [
':constants',
'//gerrit-antlr:query_exception',
'//gerrit-common:server',
'//gerrit-extension-api:api',
@ -30,6 +43,9 @@ java_library2(
'//lib/jgit:jgit',
'//lib/jgit:jgit-servlet',
'//lib/log:api',
'//lib/lucene:analyzers-common',
'//lib/lucene:core',
'//lib/lucene:query-parser',
],
compile_deps = ['//lib:servlet-api-3_0'],
visibility = ['PUBLIC'],

3
gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java
View File

@ -30,6 +30,7 @@ import com.google.gerrit.httpd.rpc.account.AccountsRestApiServlet;
import com.google.gerrit.httpd.rpc.change.ChangesRestApiServlet;
import com.google.gerrit.httpd.rpc.change.DeprecatedChangeQueryServlet;
import com.google.gerrit.httpd.rpc.config.ConfigRestApiServlet;
import com.google.gerrit.httpd.rpc.doc.QueryDocumentationFilter;
import com.google.gerrit.httpd.rpc.group.GroupsRestApiServlet;
import com.google.gerrit.httpd.rpc.project.ProjectsRestApiServlet;
import com.google.gerrit.reviewdb.client.Change;
@ -110,6 +111,8 @@ class UrlModule extends ServletModule {
serveRegex("^/(?:a/)?groups/(.*)?$").with(GroupsRestApiServlet.class);
serveRegex("^/(?:a/)?projects/(.*)?$").with(ProjectsRestApiServlet.class);
filter("/Documentation/").through(QueryDocumentationFilter.class);
if (cfg.deprecatedQuery) {
serve("/query").with(DeprecatedChangeQueryServlet.class);
}

2
gerrit-httpd/src/main/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
View File

@ -607,7 +607,7 @@ public class RestApiServlet extends HttpServlet {
throw new InstantiationException("Cannot make " + type);
}
private static void replyJson(@Nullable HttpServletRequest req,
public static void replyJson(@Nullable HttpServletRequest req,
HttpServletResponse res,
Multimap<String, String> config,
Object result)

24
gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/Constants.java Normal file
View File

@ -0,0 +1,24 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.rpc.doc;
public class Constants {
public static final String DOC_FIELD = "doc";
public static final String TITLE_FIELD = "title";
public static final String URL_FIELD = "url";
private Constants() {}
}

191
gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/doc/QueryDocumentationFilter.java Normal file
View File

@ -0,0 +1,191 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.rpc.doc;
import com.google.common.base.Strings;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.gerrit.httpd.restapi.RestApiServlet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@Singleton
public class QueryDocumentationFilter implements Filter {
private static final Logger log =
LoggerFactory.getLogger(QueryDocumentationFilter.class);
private static final String INDEX_PATH = "index.zip";
private static final Version LUCENE_VERSION = Version.LUCENE_44;
private IndexSearcher searcher;
private QueryParser parser;
protected static class DocResult {
public String title;
public String url;
public String content;
}
@Inject
QueryDocumentationFilter() {
}
@Override
public void init(FilterConfig filterConfig) {
try {
Directory dir = readIndexDirectory();
if (dir == null) {
searcher = null;
parser = null;
return;
}
IndexReader reader = DirectoryReader.open(dir);
searcher = new IndexSearcher(reader);
StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
parser = new QueryParser(LUCENE_VERSION, Constants.DOC_FIELD, analyzer);
} catch (IOException e) {
log.error("Cannot initialize documentation full text index", e);
searcher = null;
parser = null;
}
}
@Override
public void destroy() {
}
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
throws IOException, ServletException {
HttpServletRequest req = (HttpServletRequest) request;
if ("GET".equals(req.getMethod())
&& !Strings.isNullOrEmpty(req.getParameter("q"))) {
HttpServletResponse rsp = (HttpServletResponse) response;
try {
List<DocResult> result = doQuery(request.getParameter("q"));
Multimap<String, String> config = LinkedHashMultimap.create();
RestApiServlet.replyJson(req, rsp, config, result);
} catch (DocQueryException e) {
rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
} else {
chain.doFilter(request, response);
}
}
private List<DocResult> doQuery(String q) throws DocQueryException {
if (parser == null || searcher == null) {
throw new DocQueryException("Not initialized");
}
try {
Query query = parser.parse(q);
TopDocs results = searcher.search(query, Integer.MAX_VALUE);
ScoreDoc[] hits = results.scoreDocs;
int totalHits = results.totalHits;
List<DocResult> out = Lists.newArrayListWithCapacity(totalHits);
for (int i = 0; i < totalHits; i++) {
DocResult result = new DocResult();
Document doc = searcher.doc(hits[i].doc);
result.url = doc.get(Constants.URL_FIELD);
result.title = doc.get(Constants.TITLE_FIELD);
out.add(result);
}
return out;
} catch (IOException e) {
throw new DocQueryException(e);
} catch (ParseException e) {
throw new DocQueryException(e);
}
}
protected Directory readIndexDirectory() throws IOException {
Directory dir = new RAMDirectory();
byte[] buffer = new byte[4096];
InputStream index =
QueryDocumentationFilter.class.getClassLoader().getResourceAsStream(INDEX_PATH);
if (index == null) {
log.warn("No index available");
return null;
}
ZipInputStream zip = new ZipInputStream(index);
try {
ZipEntry entry;
while ((entry = zip.getNextEntry()) != null) {
IndexOutput out = dir.createOutput(entry.getName(), null);
int count;
while ((count = zip.read(buffer)) != -1) {
out.writeBytes(buffer, count);
}
out.close();
}
} finally {
zip.close();
}
// We must NOT call dir.close() here, as DirectoryReader.open() expects an opened directory.
return dir;
}
private static class DocQueryException extends Exception {
public DocQueryException() {
}
public DocQueryException(String msg) {
super(msg);
}
public DocQueryException(String msg, Throwable e) {
super(msg, e);
}
public DocQueryException(Throwable e) {
super(e);
}
}
}

1
lib/asciidoctor/BUCK
View File

@ -31,6 +31,7 @@ java_library(
srcs = ['java/DocIndexer.java'],
deps = [
':asciidoc_lib',
'//gerrit-httpd:constants',
'//lib:args4j',
'//lib:guava',
'//lib/lucene:analyzers-common',

11
lib/asciidoctor/java/DocIndexer.java
View File

@ -13,6 +13,7 @@
// limitations under the License.
import com.google.common.io.Files;
import com.google.gerrit.httpd.rpc.doc.Constants;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharArraySet;
@ -25,6 +26,7 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.util.Version;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
@ -43,9 +45,6 @@ import java.util.zip.ZipOutputStream;
public class DocIndexer {
private static final Version LUCENE_VERSION = Version.LUCENE_44;
private static final String DOC_FIELD = "doc";
private static final String URL_FIELD = "url";
private static final String TITLE_FIELD = "title";
@Option(name = "-z", usage = "output zip file")
private String zipFile;
@ -100,10 +99,10 @@ public class DocIndexer {
inputFile, inExt, outExt);
FileReader reader = new FileReader(file);
Document doc = new Document();
doc.add(new TextField(DOC_FIELD, reader));
doc.add(new TextField(Constants.DOC_FIELD, reader));
doc.add(new StringField(
URL_FIELD, prefix + outputFile, Field.Store.YES));
doc.add(new TextField(TITLE_FIELD, title, Field.Store.YES));
Constants.URL_FIELD, prefix + outputFile, Field.Store.YES));
doc.add(new TextField(Constants.TITLE_FIELD, title, Field.Store.YES));
iwriter.addDocument(doc);
reader.close();
}

8
lib/lucene/BUCK
View File

@ -40,6 +40,14 @@ maven_jar(
license = 'Apache2.0',
)
maven_jar(
name = 'query-parser',
id = 'org.apache.lucene:lucene-queryparser:4.4.0',
bin_sha1 = 'e2fca26d9c64f3aad7b8a3461dbab14782107a06',
src_sha1 = 'f23e42ab90b5b7eb888d394282eba65362e88606',
license = 'Apache2.0',
)
maven_jar(
name = 'spellchecker',
id = 'org.apache.lucene:lucene-spellchecker:3.6.2',

18
tools/build.defs
View File

@ -14,7 +14,12 @@
# These definitions support building a runnable version of Gerrit.
DOCS = ['//Documentation:html.zip']
DOCS_SRC = genfile('Documentation/html.zip')
DOCS_LIB = '//Documentation:index_lib'
DOCS_DEP = [
'//Documentation:html',
'//Documentation:index_lib',
]
LIBS = [
'//gerrit-war:log4j-config',
'//gerrit-war:init',
@ -36,7 +41,8 @@ def war(
libs = [],
pgmlibs = [],
context = [],
visibility = []
visibility = [],
docs = False
):
cmd = ['$(exe //tools:pack_war)', '-o', '$OUT', '--tmp', '$TMP']
for l in libs:
@ -46,6 +52,10 @@ def war(
src = []
dep = []
if docs:
src.append(DOCS_SRC)
dep.extend(DOCS_DEP)
cmd.extend(['--lib', DOCS_LIB])
if context:
root = get_base_path()
if root:
@ -56,6 +66,7 @@ def war(
r = root + r[2:]
r = r.replace(':', '/')
src.append(genfile(r))
if src:
cmd.append('$SRCS')
genrule(
@ -67,7 +78,7 @@ def war(
visibility = visibility,
)
def gerrit_war(name, ui = 'ui_optdbg', context = []):
def gerrit_war(name, ui = 'ui_optdbg', context = [], docs = False):
war(
name = name,
libs = LIBS + ['//gerrit-war:version'],
@ -77,4 +88,5 @@ def gerrit_war(name, ui = 'ui_optdbg', context = []):
'//gerrit-war:webapp_assets.zip',
'//gerrit-gwtui:' + ui + '.zip',
] + context,
docs = docs,
)

1
tools/eclipse/BUCK
View File

@ -13,5 +13,6 @@ java_library(
'//lib/asciidoctor:asciidoc_lib',
'//lib/asciidoctor:doc_indexer_lib',
'//lib/prolog:compiler_lib',
'//Documentation:index_lib',
] + scan_plugins(),
)