opendev/gerrit
Code Issues Proposed changes

Elasticsearch: replace native API in prod w/ REST

Browse Source 
Remove jest API which depends on Elasticsearch native API (also
removed), so no longer depend on lucene for this scope. Replace such
removed dependencies with the Elasticsearch low-level API, which does
not depend on Lucene. That now used API is a REST client with minimal
dependencies, indeed. -As opposed to Elastic's current high-level API,
which still depends on lucene and other Gerrit-constraining libraries.

Do so in order to decouple the elasticsearch client from lucene in
Gerrit. That REST client does not logically require Lucene anyway. Most
importantly, such decoupling now enables upcoming support for more than
just one Elasticsearch server version. This includes the new possibility
of bumping the latter to multiple yet later versions, such as 5 and 6.x.
The currently used version (2.4) should also still be kept, for a while.

Bumping such versions will likely require some Elasticsearch client code
adaptations, so that Gerrit can (dynamically?) switch between either
Elasticsearch server version to eventually support (hopefully soon).

Doing the same for the elasticsearch tests in Gerrit is to be done using
another change or more changes.

Add a partial and customized fork of [1], based on [1]'s commit [2], to
preserve the ability of building proper json requests for Elasticsearch.
Put that forked json-generating code under a new 'builders' sub-package.
There should be a possibility in some near future to consider removing
that fork, based on potential progress such as the one proposed in [3].
Meanwhile, this fork shall be maintained to usual Gerrit quality levels.

[1] https://github.com/elastic/elasticsearch
[2] tag: v2.4.4
[3] https://github.com/elastic/elasticsearch/issues/30791

Bug: Issue 6094
Change-Id: I720c9885c9eab2388acc328eecb9eaa6940ced0c
This commit is contained in:
Marco Miller 2018-04-27 17:23:46 -04:00 committed by David Pursehouse
parent 172cc35d2d
commit f9758fd8bb
29 changed files with 1214 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
WORKSPACE
View File

@ -918,21 +918,6 @@ maven_jar(
sha1 = "d2954e1173a608a9711f132d1768a676a8b1fb81",
)
# Java REST client for Elasticsearch.
JEST_VERSION = "2.4.0"
maven_jar(
name = "jest_common",
artifact = "io.searchbox:jest-common:" + JEST_VERSION,
sha1 = "ea779ebe7c438a53dce431f85b0d4e1d8faee2ac",
)
maven_jar(
name = "jest",
artifact = "io.searchbox:jest:" + JEST_VERSION,
sha1 = "e2a604a584e6633545ac6b1fe99ef888ab96dae9",
)
maven_jar(
name = "joda_time",
artifact = "joda-time:joda-time:2.9.9",
@ -945,6 +930,12 @@ maven_jar(
sha1 = "675642ac208e0b741bc9118dcbcae44c271b992a",
)
maven_jar(
name = "elasticsearch-rest-client",
artifact = "org.elasticsearch.client:elasticsearch-rest-client:5.6.9",
sha1 = "895706412e2fba3f842fca82ec3dece1cb4ee7d1",
)
maven_jar(
name = "compress_lzf",
artifact = "com.ning:compress-lzf:1.0.2",

212
java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@ -16,18 +16,21 @@ package com.google.gerrit.elasticsearch;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toList;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Streams;
import com.google.common.io.CharStreams;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.elasticsearch.builders.XContentBuilder;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Index;
@ -46,39 +49,50 @@ import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.protobuf.ProtobufCodec;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.core.Bulk;
import io.searchbox.core.Delete;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.IndicesExists;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Timestamp;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private static final Logger log = LoggerFactory.getLogger(AbstractElasticIndex.class);
protected static final String BULK = "_bulk";
protected static final String DELETE = "delete";
protected static final String IGNORE_UNMAPPED = "ignore_unmapped";
protected static final String INDEX = "index";
protected static final String ORDER = "order";
protected static final String SEARCH = "_search";
protected static <T> List<T> decodeProtos(
JsonObject doc, String fieldName, ProtobufCodec<T> codec) {
JsonArray field = doc.getAsJsonArray(fieldName);
@ -90,12 +104,24 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
.toList();
}
static String getContent(Response response) throws IOException {
HttpEntity responseEntity = response.getEntity();
String content = "";
if (responseEntity != null) {
InputStream contentStream = responseEntity.getContent();
try (Reader reader = new InputStreamReader(contentStream)) {
content = CharStreams.toString(reader);
}
}
return content;
}
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
private final RestClient client;
protected final String indexName;
protected final JestHttpClient client;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
@ -103,7 +129,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Schema<V> schema,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
String indexName) {
this.sitePaths = sitePaths;
this.schema = schema;
@ -126,7 +152,11 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void close() {
client.shutdownClient();
try {
client.close();
} catch (IOException e) {
// Ignored.
}
}
@Override
@ -136,60 +166,57 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void delete(K c) throws IOException {
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String uri = getURI(indexNameRaw, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, addActions(c), uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to delete change %s in index %s: %s",
c, indexName, result.getErrorMessage()));
String.format("Failed to delete change %s in index %s: %s", c, indexName, statusCode));
}
}
@Override
public void deleteAll() throws IOException {
// Delete the index, if it exists.
JestResult result = client.execute(new IndicesExists.Builder(indexName).build());
if (result.isSucceeded()) {
result = client.execute(new DeleteIndex.Builder(indexName).build());
if (!result.isSucceeded()) {
Response response = client.performRequest(HttpHead.METHOD_NAME, indexName);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
response = client.performRequest(HttpDelete.METHOD_NAME, indexName);
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format("Failed to delete index %s: %s", indexName, result.getErrorMessage()));
String.format("Failed to delete index %s: %s", indexName, statusCode));
}
}
// Recreate the index.
result = client.execute(new CreateIndex.Builder(indexName).settings(getMappings()).build());
if (!result.isSucceeded()) {
String error =
String.format("Failed to create index %s: %s", indexName, result.getErrorMessage());
response =
performRequest(HttpPut.METHOD_NAME, getMappings(), indexName, Collections.emptyMap());
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
String error = String.format("Failed to create index %s: %s", indexName, statusCode);
throw new IOException(error);
}
}
protected abstract Bulk.Builder addActions(Bulk.Builder builder, K c);
protected abstract String addActions(K c);
protected abstract String getMappings();
protected abstract String getId(V v);
protected Delete delete(String type, K c) {
protected String delete(String type, K c) {
String id = c.toString();
return new Delete.Builder(id).index(indexName).type(type).build();
}
protected io.searchbox.core.Index insert(String type, V v) throws IOException {
String id = getId(v);
String doc = toDocument(v);
return new io.searchbox.core.Index.Builder(doc).index(indexName).type(type).id(id).build();
return toAction(type, id, DELETE);
}
private static boolean shouldAddElement(Object element) {
return !(element instanceof String) || !((String) element).isEmpty();
}
private String toDocument(V v) throws IOException {
try (XContentBuilder builder = jsonBuilder().startObject()) {
protected String toDoc(V v) throws IOException {
try (XContentBuilder closeable = new XContentBuilder()) {
XContentBuilder builder = closeable.startObject();
for (Values<V> values : schema.buildFields(v)) {
String name = values.getField().getName();
if (values.getField().isRepeatable()) {
@ -205,7 +232,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
}
}
return builder.endObject().string();
return builder.endObject().string() + System.lineSeparator();
}
}
@ -214,16 +241,14 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected FieldBundle toFieldBundle(JsonObject doc) {
Map<String, FieldDef<V, ?>> allFields = getSchema().getFields();
ListMultimap<String, Object> rawFields = ArrayListMultimap.create();
for (Entry<String, JsonElement> element : doc.get("fields").getAsJsonObject().entrySet()) {
for (Map.Entry<String, JsonElement> element : doc.get("fields").getAsJsonObject().entrySet()) {
checkArgument(
allFields.containsKey(element.getKey()), "Unrecognized field " + element.getKey());
FieldType<?> type = allFields.get(element.getKey()).getType();
Iterable<JsonElement> innerItems =
element.getValue().isJsonArray()
? element.getValue().getAsJsonArray()
: Collections.singleton(element.getValue());
for (JsonElement inner : innerItems) {
if (type == FieldType.EXACT || type == FieldType.FULL_TEXT || type == FieldType.PREFIX) {
rawFields.put(element.getKey(), inner.getAsString());
@ -243,19 +268,66 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
return new FieldBundle(rawFields);
}
protected String toAction(String type, String id, String action) {
JsonObject properties = new JsonObject();
properties.addProperty("_id", id);
properties.addProperty("_index", indexName);
properties.addProperty("_type", type);
JsonObject jsonAction = new JsonObject();
jsonAction.add(action, properties);
return jsonAction.toString() + System.lineSeparator();
}
protected void addNamedElement(String name, JsonObject element, JsonArray array) {
JsonObject arrayElement = new JsonObject();
arrayElement.add(name, element);
array.add(arrayElement);
}
protected Map<String, String> getRefreshParam() {
Map<String, String> params = new HashMap<>();
params.put("refresh", "true");
return params;
}
protected String getSearch(SearchSourceBuilder searchSource, JsonArray sortArray) {
JsonObject search = new JsonParser().parse(searchSource.toString()).getAsJsonObject();
search.add("sort", sortArray);
return gson.toJson(search);
}
protected JsonArray getSortArray(String idFieldName) {
JsonObject properties = new JsonObject();
properties.addProperty(ORDER, "asc");
properties.addProperty(IGNORE_UNMAPPED, true);
JsonArray sortArray = new JsonArray();
addNamedElement(idFieldName, properties, sortArray);
return sortArray;
}
protected String getURI(String type, String request) throws UnsupportedEncodingException {
String encodedType = URLEncoder.encode(type, UTF_8.toString());
String encodedIndexName = URLEncoder.encode(indexName, UTF_8.toString());
return encodedIndexName + "/" + encodedType + "/" + request;
}
protected Response performRequest(
String method, String payload, String uri, Map<String, String> params) throws IOException {
HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
return client.performRequest(method, uri, params, entity);
}
protected class ElasticQuerySource implements DataSource<V> {
private final QueryOptions opts;
private final Search search;
private final String search;
private final String index;
ElasticQuerySource(Predicate<V> p, QueryOptions opts, String type, Sort sort)
throws QueryParseException {
this(p, opts, ImmutableList.of(type), ImmutableList.of(sort));
}
ElasticQuerySource(
Predicate<V> p, QueryOptions opts, Collection<String> types, Collection<Sort> sorts)
ElasticQuerySource(Predicate<V> p, QueryOptions opts, String index, JsonArray sortArray)
throws QueryParseException {
this.opts = opts;
this.index = index;
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
@ -263,13 +335,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(opts.fields()));
search =
new Search.Builder(searchSource.toString())
.addType(types)
.addSort(sorts)
.addIndex(indexName)
.build();
search = getSearch(searchSource, sortArray);
}
@Override
@ -287,17 +353,17 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
return readImpl(AbstractElasticIndex.this::toFieldBundle);
}
@Override
public String toString() {
return search.toString();
}
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) throws OrmException {
try {
List<T> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(index, SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
@ -309,7 +375,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<T> r = Collections.unmodifiableList(results);
return new ResultSet<T>() {

11
java/com/google/gerrit/elasticsearch/BUILD
View File

@ -16,15 +16,16 @@ java_library(
"//lib:protobuf",
"//lib/commons:codec",
"//lib/commons:lang",
"//lib/elasticsearch",
"//lib/elasticsearch:joda-time",
"//lib/elasticsearch-rest-client",
"//lib/guice",
"//lib/guice:guice-assistedinject",
"//lib/jest",
"//lib/jest:jest-common",
"//lib/httpcomponents:httpasyncclient",
"//lib/httpcomponents:httpclient",
"//lib/httpcomponents:httpcore",
"//lib/httpcomponents:httpcore-nio",
"//lib/jackson:jackson-core",
"//lib/jgit/org.eclipse.jgit:jgit",
"//lib/log:api",
"//lib/lucene:lucene-analyzers-common",
"//lib/lucene:lucene-core",
],
)

39
java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@ -31,19 +31,18 @@ import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountField;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
implements AccountIndex {
@ -65,7 +64,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<AccountState> schema) {
super(cfg, sitePaths, schema, clientBuilder, ACCOUNTS);
this.accountCache = accountCache;
@ -74,33 +73,31 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@Override
public void replace(AccountState as) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(ACCOUNTS)
.addAction(insert(ACCOUNTS, as))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(ACCOUNTS, getId(as), INDEX);
bulk += toDoc(as);
String uri = getURI(ACCOUNTS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace account %s in index %s: %s",
as.getAccount().getId(), indexName, result.getErrorMessage()));
as.getAccount().getId(), indexName, statusCode));
}
}
@Override
public DataSource<AccountState> getSource(Predicate<AccountState> p, QueryOptions opts)
throws QueryParseException {
Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::accountFields), ACCOUNTS, sort);
JsonArray sortArray = getSortArray(AccountField.ID.getName());
return new ElasticQuerySource(
p, opts.filterFields(IndexUtils::accountFields), ACCOUNTS, sortArray);
}
@Override
protected Builder addActions(Builder builder, Account.Id c) {
return builder.addAction(delete(ACCOUNTS, c));
protected String addActions(Account.Id c) {
return delete(ACCOUNTS, c);
}
@Override

60
java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@ -24,7 +24,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@ -61,18 +60,16 @@ import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
/** Secondary index implementation using Elasticsearch. */
class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
@ -102,7 +99,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
SitePaths sitePaths,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<ChangeData> schema) {
super(cfg, sitePaths, schema, clientBuilder, CHANGES);
this.db = db;
@ -127,20 +124,17 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
throw new IOException(e);
}
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType("changes")
.addAction(insert(insertIndex, cd))
.addAction(delete(deleteIndex, cd.getId()))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(insertIndex, getId(cd), INDEX);
bulk += toDoc(cd);
bulk += toAction(deleteIndex, cd.getId().toString(), DELETE);
String uri = getURI(CHANGES, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace change %s in index %s: %s",
cd.getId(), indexName, result.getErrorMessage()));
"Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode));
}
}
@ -156,20 +150,28 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
indexes.add(CLOSED_CHANGES);
}
List<Sort> sorts =
ImmutableList.of(
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
for (Sort sort : sorts) {
sort.setIgnoreUnmapped();
}
QueryOptions filteredOpts = opts.filterFields(IndexUtils::changeFields);
return new ElasticQuerySource(p, filteredOpts, indexes, sorts);
return new ElasticQuerySource(p, filteredOpts, getURI(indexes), getSortArray());
}
private JsonArray getSortArray() {
JsonObject properties = new JsonObject();
properties.addProperty(ORDER, "desc");
properties.addProperty(IGNORE_UNMAPPED, true);
JsonArray sortArray = new JsonArray();
addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
addNamedElement(ChangeField.LEGACY_ID.getName(), properties, sortArray);
return sortArray;
}
private String getURI(List<String> types) {
return String.join(",", types);
}
@Override
protected Builder addActions(Builder builder, Id c) {
return builder.addAction(delete(OPEN_CHANGES, c)).addAction(delete(OPEN_CHANGES, c));
protected String addActions(Id c) {
return delete(OPEN_CHANGES, c) + delete(CLOSED_CHANGES, c);
}
@Override

30
java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
View File

@ -18,13 +18,12 @@ import com.google.common.base.MoreObjects;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpHost;
import org.eclipse.jgit.lib.Config;
@Singleton
@ -33,7 +32,7 @@ class ElasticConfiguration {
private static final String DEFAULT_PORT = "9200";
private static final String DEFAULT_PROTOCOL = "http";
final List<String> urls;
final List<HttpHost> urls;
final String username;
final String password;
final boolean requestCompression;
@ -59,14 +58,18 @@ class ElasticConfiguration {
Set<String> subsections = cfg.getSubsections("elasticsearch");
if (subsections.isEmpty()) {
this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT));
HttpHost httpHost =
new HttpHost(DEFAULT_HOST, Integer.valueOf(DEFAULT_PORT), DEFAULT_PROTOCOL);
this.urls = Collections.singletonList(httpHost);
} else {
this.urls = new ArrayList<>(subsections.size());
for (String subsection : subsections) {
String port = getString(cfg, subsection, "port", DEFAULT_PORT);
String host = getString(cfg, subsection, "hostname", DEFAULT_HOST);
String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL);
this.urls.add(buildUrl(protocol, host, port));
HttpHost httpHost = new HttpHost(host, Integer.valueOf(port), protocol);
this.urls.add(httpHost);
}
}
}
@ -74,19 +77,4 @@ class ElasticConfiguration {
private String getString(Config cfg, String subsection, String name, String defaultValue) {
return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
}
private String buildUrl(String protocol, String hostname, String port) {
try {
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
} catch (MalformedURLException | NumberFormatException e) {
throw new RuntimeException(
"Cannot build url to Elasticsearch from values: protocol="
+ protocol
+ " hostname="
+ hostname
+ " port="
+ port,
e);
}
}
}

37
java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@ -29,18 +29,18 @@ import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.group.GroupField;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.search.sort.Sort;
import java.io.IOException;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, InternalGroup>
implements GroupIndex {
@ -62,7 +62,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<InternalGroup> schema) {
super(cfg, sitePaths, schema, clientBuilder, GROUPS);
this.groupCache = groupCache;
@ -71,33 +71,30 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@Override
public void replace(InternalGroup group) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(GROUPS)
.addAction(insert(GROUPS, group))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(GROUPS, getId(group), INDEX);
bulk += toDoc(group);
String uri = getURI(GROUPS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace group %s in index %s: %s",
group.getGroupUUID().get(), indexName, result.getErrorMessage()));
group.getGroupUUID().get(), indexName, statusCode));
}
}
@Override
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
Sort sort = new Sort(GroupField.UUID.getName(), Sort.Sorting.ASC);
sort.setIgnoreUnmapped();
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), GROUPS, sort);
JsonArray sortArray = getSortArray(GroupField.UUID.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), GROUPS, sortArray);
}
@Override
protected Builder addActions(Builder builder, AccountGroup.UUID c) {
return builder.addAction(delete(GROUPS, c));
protected String addActions(AccountGroup.UUID c) {
return delete(GROUPS, c);
}
@Override

22
java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
View File

@ -16,31 +16,37 @@ package com.google.gerrit.elasticsearch;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.indices.aliases.GetAliases;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
@Singleton
class ElasticIndexVersionDiscovery {
private final JestHttpClient client;
private final RestClient client;
@Inject
ElasticIndexVersionDiscovery(JestClientBuilder clientBuilder) {
ElasticIndexVersionDiscovery(ElasticRestClientBuilder clientBuilder) {
this.client = clientBuilder.build();
}
List<String> discover(String prefix, String indexName) throws IOException {
String name = prefix + indexName + "_";
JestResult result = client.execute(new GetAliases.Builder().addIndex(name + "*").build());
if (result.isSucceeded()) {
JsonObject object = result.getJsonObject().getAsJsonObject();
Response response = client.performRequest(HttpGet.METHOD_NAME, name + "*/_aliases");
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
String content = AbstractElasticIndex.getContent(response);
JsonObject object = new JsonParser().parse(content).getAsJsonObject();
List<String> versions = new ArrayList<>(object.size());
for (Entry<String, JsonElement> entry : object.entrySet()) {
versions.add(entry.getKey().replace(name, ""));

39
java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
View File

@ -29,19 +29,18 @@ import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, ProjectData>
implements ProjectIndex {
@ -63,7 +62,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<ProjectData> schema) {
super(cfg, sitePaths, schema, clientBuilder, PROJECTS);
this.projectCache = projectCache;
@ -72,33 +71,31 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
@Override
public void replace(ProjectData projectState) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(PROJECTS)
.addAction(insert(PROJECTS, projectState))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(PROJECTS, getId(projectState), INDEX);
bulk += toDoc(projectState);
String uri = getURI(PROJECTS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace project %s in index %s: %s",
projectState.getProject().getName(), indexName, result.getErrorMessage()));
projectState.getProject().getName(), indexName, statusCode));
}
}
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), PROJECTS, sort);
JsonArray sortArray = getSortArray(ProjectField.NAME.getName());
return new ElasticQuerySource(
p, opts.filterFields(IndexUtils::projectFields), PROJECTS, sortArray);
}
@Override
protected Builder addActions(Builder builder, Project.NameKey nameKey) {
return builder.addAction(delete(PROJECTS, nameKey));
protected String addActions(Project.NameKey nameKey) {
return delete(PROJECTS, nameKey);
}
@Override

31
java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
View File

@ -14,6 +14,9 @@
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.builders.BoolQueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilders;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.query.AndPredicate;
@ -28,10 +31,6 @@ import com.google.gerrit.index.query.RegexPredicate;
import com.google.gerrit.index.query.TimestampRangePredicate;
import com.google.gerrit.server.query.change.AfterPredicate;
import java.time.Instant;
import org.apache.lucene.search.BooleanQuery;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
public class ElasticQueryBuilder {
@ -52,27 +51,19 @@ public class ElasticQueryBuilder {
}
private <T> BoolQueryBuilder and(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
}
private <T> BoolQueryBuilder or(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
}
private <T> QueryBuilder not(Predicate<T> p) throws QueryParseException {

58
java/com/google/gerrit/elasticsearch/ElasticRestClientBuilder.java Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
@Singleton
class ElasticRestClientBuilder {
private final HttpHost[] hosts;
private final String username;
private final String password;
@Inject
ElasticRestClientBuilder(ElasticConfiguration cfg) {
hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]);
username = cfg.username;
password = cfg.password;
}
RestClient build() {
RestClientBuilder builder = RestClient.builder(hosts);
setConfiguredCredentialsIfAny(builder);
return builder.build();
}
private void setConfiguredCredentialsIfAny(RestClientBuilder builder) {
if (username != null && password != null) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(
(HttpAsyncClientBuilder httpClientBuilder) ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
}
}

54
java/com/google/gerrit/elasticsearch/JestClientBuilder.java
View File

@ -1,54 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.client.config.HttpClientConfig.Builder;
import io.searchbox.client.http.JestHttpClient;
import java.util.concurrent.TimeUnit;
@Singleton
class JestClientBuilder {
private final ElasticConfiguration cfg;
@Inject
JestClientBuilder(ElasticConfiguration cfg) {
this.cfg = cfg;
}
JestHttpClient build() {
JestClientFactory factory = new JestClientFactory();
Builder builder =
new HttpClientConfig.Builder(cfg.urls)
.multiThreaded(true)
.discoveryEnabled(false)
.connTimeout((int) cfg.connectionTimeout)
.maxConnectionIdleTime(cfg.maxConnectionIdleTime, cfg.maxConnectionIdleUnit)
.maxTotalConnection(cfg.maxTotalConnection)
.readTimeout(cfg.readTimeout)
.requestCompressionEnabled(cfg.requestCompression)
.discoveryFrequency(1L, TimeUnit.MINUTES);
if (cfg.username != null && cfg.password != null) {
builder.defaultCredentials(cfg.username, cfg.password);
}
factory.setHttpClientConfig(builder.build());
return (JestHttpClient) factory.getObject();
}
}

88
java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java Normal file
View File

@ -0,0 +1,88 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* A Query that matches documents matching boolean combinations of other queries. A trimmed down
* version of {@link org.elasticsearch.index.query.BoolQueryBuilder} for this very package.
*/
public class BoolQueryBuilder extends QueryBuilder {
private final List<QueryBuilder> mustClauses = new ArrayList<>();
private final List<QueryBuilder> mustNotClauses = new ArrayList<>();
private final List<QueryBuilder> filterClauses = new ArrayList<>();
private final List<QueryBuilder> shouldClauses = new ArrayList<>();
/**
* Adds a query that <b>must</b> appear in the matching documents and will contribute to scoring.
*/
public BoolQueryBuilder must(QueryBuilder queryBuilder) {
mustClauses.add(queryBuilder);
return this;
}
/**
* Adds a query that <b>must not</b> appear in the matching documents and will not contribute to
* scoring.
*/
public BoolQueryBuilder mustNot(QueryBuilder queryBuilder) {
mustNotClauses.add(queryBuilder);
return this;
}
/**
* Adds a query that <i>should</i> appear in the matching documents. For a boolean query with no
* <tt>MUST</tt> clauses one or more <code>SHOULD</code> clauses must match a document for the
* BooleanQuery to match.
*/
public BoolQueryBuilder should(QueryBuilder queryBuilder) {
shouldClauses.add(queryBuilder);
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("bool");
doXArrayContent("must", mustClauses, builder);
doXArrayContent("filter", filterClauses, builder);
doXArrayContent("must_not", mustNotClauses, builder);
doXArrayContent("should", shouldClauses, builder);
builder.endObject();
}
private void doXArrayContent(String field, List<QueryBuilder> clauses, XContentBuilder builder)
throws IOException {
if (clauses.isEmpty()) {
return;
}
if (clauses.size() == 1) {
builder.field(field);
clauses.get(0).toXContent(builder);
} else {
builder.startArray(field);
for (QueryBuilder clause : clauses) {
clause.toXContent(builder);
}
builder.endArray();
}
}
}

37
java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java Normal file
View File

@ -0,0 +1,37 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* Constructs a query that only match on documents that the field has a value in them. A trimmed
* down version of {@link org.elasticsearch.index.query.ExistsQueryBuilder} for this very package.
*/
class ExistsQueryBuilder extends QueryBuilder {
private final String name;
ExistsQueryBuilder(String name) {
this.name = name;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("exists");
builder.field("field", name);
builder.endObject();
}
}

30
java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A query that matches on all documents. A trimmed down version of {@link
* org.elasticsearch.index.query.MatchAllQueryBuilder} for this very package.
*/
class MatchAllQueryBuilder extends QueryBuilder {
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("match_all");
builder.endObject();
}
}

64
java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java Normal file
View File

@ -0,0 +1,64 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.Locale;
/**
* Match query is a query that analyzes the text and constructs a query as the result of the
* analysis. It can construct different queries based on the type provided. A trimmed down version
* of {@link org.elasticsearch.index.query.MatchQueryBuilder} for this very package.
*/
class MatchQueryBuilder extends QueryBuilder {
enum Type {
/** The text is analyzed and used as a phrase query. */
PHRASE,
/** The text is analyzed and used in a phrase query, with the last term acting as a prefix. */
PHRASE_PREFIX
}
private final String name;
private final Object text;
private Type type;
/** Constructs a new text query. */
MatchQueryBuilder(String name, Object text) {
this.name = name;
this.text = text;
}
/** Sets the type of the text query. */
MatchQueryBuilder type(Type type) {
this.type = type;
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("match");
builder.startObject(name);
builder.field("query", text);
if (type != null) {
builder.field("type", type.toString().toLowerCase(Locale.ENGLISH));
}
builder.endObject();
builder.endObject();
}
}

34
java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down version of {@link org.elasticsearch.index.query.QueryBuilder} for this very
* package.
*/
public abstract class QueryBuilder {
protected QueryBuilder() {}
protected void toXContent(XContentBuilder builder) throws IOException {
builder.startObject();
doXContent(builder);
builder.endObject();
}
protected abstract void doXContent(XContentBuilder builder) throws IOException;
}

102
java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java Normal file
View File

@ -0,0 +1,102 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
/**
* A static factory for simple "import static" usage. A trimmed down version of {@link
* org.elasticsearch.index.query.QueryBuilders} for this very package.
*/
public abstract class QueryBuilders {
/** A query that match on all documents. */
public static MatchAllQueryBuilder matchAllQuery() {
return new MatchAllQueryBuilder();
}
/**
* Creates a text query with type "PHRASE" for the provided field name and text.
*
* @param name The field name.
* @param text The query text (to be analyzed).
*/
public static MatchQueryBuilder matchPhraseQuery(String name, Object text) {
return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE);
}
/**
* Creates a match query with type "PHRASE_PREFIX" for the provided field name and text.
*
* @param name The field name.
* @param text The query text (to be analyzed).
*/
public static MatchQueryBuilder matchPhrasePrefixQuery(String name, Object text) {
return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE_PREFIX);
}
/**
* A Query that matches documents containing a term.
*
* @param name The name of the field
* @param value The value of the term
*/
public static TermQueryBuilder termQuery(String name, String value) {
return new TermQueryBuilder(name, value);
}
/**
* A Query that matches documents containing a term.
*
* @param name The name of the field
* @param value The value of the term
*/
public static TermQueryBuilder termQuery(String name, int value) {
return new TermQueryBuilder(name, value);
}
/**
* A Query that matches documents within an range of terms.
*
* @param name The field name
*/
public static RangeQueryBuilder rangeQuery(String name) {
return new RangeQueryBuilder(name);
}
/**
* A Query that matches documents containing terms with a specified regular expression.
*
* @param name The name of the field
* @param regexp The regular expression
*/
public static RegexpQueryBuilder regexpQuery(String name, String regexp) {
return new RegexpQueryBuilder(name, regexp);
}
/** A Query that matches documents matching boolean combinations of other queries. */
public static BoolQueryBuilder boolQuery() {
return new BoolQueryBuilder();
}
/**
* A filter to filter only documents where a field exists in them.
*
* @param name The name of the field
*/
public static ExistsQueryBuilder existsQuery(String name) {
return new ExistsQueryBuilder(name);
}
private QueryBuilders() {}
}

35
java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java Normal file
View File

@ -0,0 +1,35 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down and further altered version of {@link
* org.elasticsearch.action.support.QuerySourceBuilder} for this very package.
*/
class QuerySourceBuilder {
private final QueryBuilder queryBuilder;
QuerySourceBuilder(QueryBuilder queryBuilder) {
this.queryBuilder = queryBuilder;
}
void innerToXContent(XContentBuilder builder) throws IOException {
builder.field("query");
queryBuilder.toXContent(builder);
}
}

88
java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java Normal file
View File

@ -0,0 +1,88 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents within an range of terms. A trimmed down version of {@link
* org.elasticsearch.index.query.RangeQueryBuilder} for this very package.
*/
public class RangeQueryBuilder extends QueryBuilder {
private final String name;
private Object from;
private Object to;
private boolean includeLower = true;
private boolean includeUpper = true;
/**
* A Query that matches documents within an range of terms.
*
* @param name The field name
*/
RangeQueryBuilder(String name) {
this.name = name;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gt(Object from) {
this.from = from;
this.includeLower = false;
return this;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gte(Object from) {
this.from = from;
this.includeLower = true;
return this;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gte(int from) {
this.from = from;
this.includeLower = true;
return this;
}
/** The to part of the range query. Null indicates unbounded. */
public RangeQueryBuilder lte(Object to) {
this.to = to;
this.includeUpper = true;
return this;
}
/** The to part of the range query. Null indicates unbounded. */
public RangeQueryBuilder lte(int to) {
this.to = to;
this.includeUpper = true;
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("range");
builder.startObject(name);
builder.field("from", from);
builder.field("to", to);
builder.field("include_lower", includeLower);
builder.field("include_upper", includeUpper);
builder.endObject();
builder.endObject();
}
}

50
java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that does fuzzy matching for a specific value. A trimmed down version of {@link
* org.elasticsearch.index.query.RegexpQueryBuilder} for this very package.
*/
class RegexpQueryBuilder extends QueryBuilder {
private final String name;
private final String regexp;
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param regexp The regular expression
*/
RegexpQueryBuilder(String name, String regexp) {
this.name = name;
this.regexp = regexp;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("regexp");
builder.startObject(name);
builder.field("value", regexp);
builder.field("flags_value", 65535);
builder.endObject();
builder.endObject();
}
}

108
java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java Normal file
View File

@ -0,0 +1,108 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.List;
/**
* A search source builder allowing to easily build search source. A trimmed down and further
* altered version of {@link org.elasticsearch.search.builder.SearchSourceBuilder} for this very
* package.
*/
public class SearchSourceBuilder {
private QuerySourceBuilder querySourceBuilder;
private int from = -1;
private int size = -1;
private List<String> fieldNames;
/** Constructs a new search source builder. */
public SearchSourceBuilder() {}
/** Constructs a new search source builder with a search query. */
public SearchSourceBuilder query(QueryBuilder query) {
if (this.querySourceBuilder == null) {
this.querySourceBuilder = new QuerySourceBuilder(query);
}
return this;
}
/** From index to start the search from. Defaults to <tt>0</tt>. */
public SearchSourceBuilder from(int from) {
this.from = from;
return this;
}
/** The number of search hits to return. Defaults to <tt>10</tt>. */
public SearchSourceBuilder size(int size) {
this.size = size;
return this;
}
/**
* Sets the fields to load and return as part of the search request. If none are specified, the
* source of the document will be returned.
*/
public SearchSourceBuilder fields(List<String> fields) {
this.fieldNames = fields;
return this;
}
@Override
public final String toString() {
try {
XContentBuilder builder = new XContentBuilder();
toXContent(builder);
return builder.string();
} catch (IOException ioe) {
return "";
}
}
private void toXContent(XContentBuilder builder) throws IOException {
builder.startObject();
innerToXContent(builder);
builder.endObject();
}
private void innerToXContent(XContentBuilder builder) throws IOException {
if (from != -1) {
builder.field("from", from);
}
if (size != -1) {
builder.field("size", size);
}
if (querySourceBuilder != null) {
querySourceBuilder.innerToXContent(builder);
}
if (fieldNames != null) {
if (fieldNames.size() == 1) {
builder.field("fields", fieldNames.get(0));
} else {
builder.startArray("fields");
for (String fieldName : fieldNames) {
builder.value(fieldName);
}
builder.endArray();
}
}
}
}

66
java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java Normal file
View File

@ -0,0 +1,66 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents containing a term. A trimmed down version of {@link
* org.elasticsearch.index.query.TermQueryBuilder} for this very package.
*/
class TermQueryBuilder extends QueryBuilder {
private final String name;
private final Object value;
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
TermQueryBuilder(String name, String value) {
this(name, (Object) value);
}
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
TermQueryBuilder(String name, int value) {
this(name, (Object) value);
}
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
private TermQueryBuilder(String name, Object value) {
this.name = name;
this.value = value;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("term");
builder.field(name, value);
builder.endObject();
}
}

170
java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java Normal file
View File

@ -0,0 +1,170 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import static java.time.format.DateTimeFormatter.ISO_INSTANT;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.google.common.base.Charsets;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.util.Date;
/**
* A trimmed down and further altered version of {@link
* org.elasticsearch.common.xcontent.XContentBuilder} for this very package.
*/
public final class XContentBuilder implements Closeable {
private final JsonGenerator generator;
private final ByteArrayOutputStream bos = new ByteArrayOutputStream();
/**
* Constructs a new builder. Make sure to call {@link #close()} when the builder is done with.
* Inspired from {@link org.elasticsearch.common.xcontent.json.JsonXContent} static block.
*/
public XContentBuilder() throws IOException {
JsonFactory jsonFactory = new JsonFactory();
jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
jsonFactory.configure(
JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW,
false); // this trips on many mappings now...
this.generator = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
}
public XContentBuilder startObject(String name) throws IOException {
field(name);
startObject();
return this;
}
public XContentBuilder startObject() throws IOException {
generator.writeStartObject();
return this;
}
public XContentBuilder endObject() throws IOException {
generator.writeEndObject();
return this;
}
public void startArray(String name) throws IOException {
field(name);
startArray();
}
private void startArray() throws IOException {
generator.writeStartArray();
}
public void endArray() throws IOException {
generator.writeEndArray();
}
public XContentBuilder field(String name) throws IOException {
generator.writeFieldName(name);
return this;
}
public XContentBuilder field(String name, String value) throws IOException {
field(name);
generator.writeString(value);
return this;
}
public XContentBuilder field(String name, int value) throws IOException {
field(name);
generator.writeNumber(value);
return this;
}
public XContentBuilder field(String name, Iterable<?> value) throws IOException {
startArray(name);
for (Object o : value) {
value(o);
}
endArray();
return this;
}
public XContentBuilder field(String name, Object value) throws IOException {
field(name);
writeValue(value);
return this;
}
public XContentBuilder value(Object value) throws IOException {
writeValue(value);
return this;
}
public XContentBuilder field(String name, boolean value) throws IOException {
field(name);
generator.writeBoolean(value);
return this;
}
public XContentBuilder value(String value) throws IOException {
generator.writeString(value);
return this;
}
@Override
public void close() {
try {
generator.close();
} catch (IOException e) {
// ignore
}
}
/** Returns a string representation of the builder (only applicable for text based xcontent). */
public String string() {
close();
byte[] bytesArray = bos.toByteArray();
return new String(bytesArray, Charsets.UTF_8);
}
private void writeValue(Object value) throws IOException {
if (value == null) {
generator.writeNull();
return;
}
Class<?> type = value.getClass();
if (type == String.class) {
generator.writeString((String) value);
} else if (type == Integer.class) {
generator.writeNumber(((Integer) value));
} else if (type == byte[].class) {
generator.writeBinary((byte[]) value);
} else if (value instanceof Date) {
generator.writeString(ISO_INSTANT.format(((Date) value).toInstant()));
} else {
// if this is a "value" object, like enum, DistanceUnit, ..., just toString it
// yea, it can be misleading when toString a Java class, but really, jackson should be used in
// that case
generator.writeString(value.toString());
// throw new ElasticsearchIllegalArgumentException("type not supported for generic value
// conversion: " + type);
}
}
}

5
lib/LICENSE-elasticsearch Normal file
View File

@ -0,0 +1,5 @@
Elasticsearch
Copyright 2009-2015 Elasticsearch
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).

8
lib/elasticsearch-rest-client/BUILD Normal file
View File

@ -0,0 +1,8 @@
package(default_visibility = ["//visibility:public"])
java_library(
name = "elasticsearch-rest-client",
data = ["//lib:LICENSE-elasticsearch"],
visibility = ["//visibility:public"],
exports = ["@elasticsearch-rest-client//jar"],
)

1
lib/elasticsearch/BUILD
View File

@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"])
java_library(
name = "elasticsearch",
testonly = 1,
data = ["//lib:LICENSE-Apache2.0"],
exports = ["@elasticsearch//jar"],
runtime_deps = [

23
lib/jest/BUILD
View File

@ -1,23 +0,0 @@
package(default_visibility = ["//visibility:public"])
java_library(
name = "jest-common",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@jest_common//jar"],
runtime_deps = [
"//lib/commons:lang3",
],
)
java_library(
name = "jest",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@jest//jar"],
runtime_deps = [
"//lib/httpcomponents:httpasyncclient",
"//lib/httpcomponents:httpclient",
"//lib/httpcomponents:httpcore-nio",
],
)

1
tools/eclipse/BUILD
View File

@ -10,6 +10,7 @@ load(
TEST_DEPS = [
"//gerrit-gwtui:ui_tests",
"//javatests/com/google/gerrit/server:server_tests",
"//lib/elasticsearch",
]
DEPS = [