opendev/gerrit
Code Issues Proposed changes

Merge branch 'stable-2.14' into stable-2.15

Browse Source 
* stable-2.14:
  ElasticContainer: Allow to specify the docker container version to create
  ElasticContainer: Include cause in AssumptionViolatedException
  ElasticContainer: Create with static method
  Acceptance tests: Replace embedded ES with docker testcontainer
  Elasticsearch: replace native API in prod w/ REST

Changes to ReindexIT done in Iccf443102 ("Acceptance tests: Replace embedded
ES with docker testcontainer") are reverted in this merge since that test
currently doesn't work with Elasticsearch on stable-2.15 (issue 8799).

Change-Id: I21d8b10ebd450c7dc840846a5a0d836b4243dacc
This commit is contained in:
David Pursehouse
2018-05-28 19:24:39 +09:00
parent e65498896f c337290165
commit d92ba3bc8d
37 changed files with 1451 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@@ -15,14 +15,17 @@
package com.google.gerrit.elasticsearch;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toList;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import com.google.common.io.CharStreams;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.elasticsearch.builders.XContentBuilder;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
@@ -33,20 +36,39 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.protobuf.ProtobufCodec;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.core.Bulk;
import io.searchbox.core.Delete;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.IndicesExists;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected static final String BULK = "_bulk";
protected static final String DELETE = "delete";
protected static final String IGNORE_UNMAPPED = "ignore_unmapped";
protected static final String INDEX = "index";
protected static final String ORDER = "order";
protected static final String SEARCH = "_search";
protected static <T> List<T> decodeProtos(
JsonObject doc, String fieldName, ProtobufCodec<T> codec) {
JsonArray field = doc.getAsJsonArray(fieldName);
@@ -58,12 +80,24 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
.toList();
}
static String getContent(Response response) throws IOException {
HttpEntity responseEntity = response.getEntity();
String content = "";
if (responseEntity != null) {
InputStream contentStream = responseEntity.getContent();
try (Reader reader = new InputStreamReader(contentStream)) {
content = CharStreams.toString(reader);
}
}
return content;
}
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
private final RestClient client;
protected final String indexName;
protected final JestHttpClient client;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
@@ -71,7 +105,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Schema<V> schema,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
String indexName) {
this.sitePaths = sitePaths;
this.schema = schema;
@@ -94,7 +128,11 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void close() {
client.shutdownClient();
try {
client.close();
} catch (IOException e) {
// Ignored.
}
}
@Override
@@ -104,60 +142,57 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void delete(K c) throws IOException {
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String uri = getURI(indexNameRaw, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, addActions(c), uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to delete change %s in index %s: %s",
c, indexName, result.getErrorMessage()));
String.format("Failed to delete change %s in index %s: %s", c, indexName, statusCode));
}
}
@Override
public void deleteAll() throws IOException {
// Delete the index, if it exists.
JestResult result = client.execute(new IndicesExists.Builder(indexName).build());
if (result.isSucceeded()) {
result = client.execute(new DeleteIndex.Builder(indexName).build());
if (!result.isSucceeded()) {
Response response = client.performRequest(HttpHead.METHOD_NAME, indexName);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
response = client.performRequest(HttpDelete.METHOD_NAME, indexName);
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format("Failed to delete index %s: %s", indexName, result.getErrorMessage()));
String.format("Failed to delete index %s: %s", indexName, statusCode));
}
}
// Recreate the index.
result = client.execute(new CreateIndex.Builder(indexName).settings(getMappings()).build());
if (!result.isSucceeded()) {
String error =
String.format("Failed to create index %s: %s", indexName, result.getErrorMessage());
response =
performRequest(HttpPut.METHOD_NAME, getMappings(), indexName, Collections.emptyMap());
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
String error = String.format("Failed to create index %s: %s", indexName, statusCode);
throw new IOException(error);
}
}
protected abstract Bulk.Builder addActions(Bulk.Builder builder, K c);
protected abstract String addActions(K c);
protected abstract String getMappings();
protected abstract String getId(V v);
protected Delete delete(String type, K c) {
protected String delete(String type, K c) {
String id = c.toString();
return new Delete.Builder(id).index(indexName).type(type).build();
}
protected io.searchbox.core.Index insert(String type, V v) throws IOException {
String id = getId(v);
String doc = toDoc(v);
return new io.searchbox.core.Index.Builder(doc).index(indexName).type(type).id(id).build();
return toAction(type, id, DELETE);
}
private static boolean shouldAddElement(Object element) {
return !(element instanceof String) || !((String) element).isEmpty();
}
private String toDoc(V v) throws IOException {
try (XContentBuilder builder = jsonBuilder().startObject()) {
protected String toDoc(V v) throws IOException {
try (XContentBuilder closeable = new XContentBuilder()) {
XContentBuilder builder = closeable.startObject();
for (Values<V> values : schema.buildFields(v)) {
String name = values.getField().getName();
if (values.getField().isRepeatable()) {
@@ -173,7 +208,58 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
}
}
return builder.endObject().string();
return builder.endObject().string() + System.lineSeparator();
}
}
protected String toAction(String type, String id, String action) {
JsonObject properties = new JsonObject();
properties.addProperty("_id", id);
properties.addProperty("_index", indexName);
properties.addProperty("_type", type);
JsonObject jsonAction = new JsonObject();
jsonAction.add(action, properties);
return jsonAction.toString() + System.lineSeparator();
}
protected void addNamedElement(String name, JsonObject element, JsonArray array) {
JsonObject arrayElement = new JsonObject();
arrayElement.add(name, element);
array.add(arrayElement);
}
protected Map<String, String> getRefreshParam() {
Map<String, String> params = new HashMap<>();
params.put("refresh", "true");
return params;
}
protected String getSearch(SearchSourceBuilder searchSource, JsonArray sortArray) {
JsonObject search = new JsonParser().parse(searchSource.toString()).getAsJsonObject();
search.add("sort", sortArray);
return gson.toJson(search);
}
protected JsonArray getSortArray(String idFieldName) {
JsonObject properties = new JsonObject();
properties.addProperty(ORDER, "asc");
properties.addProperty(IGNORE_UNMAPPED, true);
JsonArray sortArray = new JsonArray();
addNamedElement(idFieldName, properties, sortArray);
return sortArray;
}
protected String getURI(String type, String request) throws UnsupportedEncodingException {
String encodedType = URLEncoder.encode(type, UTF_8.toString());
String encodedIndexName = URLEncoder.encode(indexName, UTF_8.toString());
return encodedIndexName + "/" + encodedType + "/" + request;
}
protected Response performRequest(
String method, String payload, String uri, Map<String, String> params) throws IOException {
HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
return client.performRequest(method, uri, params, entity);
}
}

81
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@@ -16,10 +16,11 @@ package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.account.AccountField.ID;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
@@ -36,51 +37,48 @@ import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
implements AccountIndex {
static class AccountMapping {
public static class AccountMapping {
MappingProperties accounts;
AccountMapping(Schema<AccountState> schema) {
public AccountMapping(Schema<AccountState> schema) {
this.accounts = ElasticMapping.createMapping(schema);
}
}
static final String ACCOUNTS = "accounts";
public static final String ACCOUNTS = "accounts";
private static final Logger log = LoggerFactory.getLogger(ElasticAccountIndex.class);
private final AccountMapping mapping;
private final Provider<AccountCache> accountCache;
@Inject
@AssistedInject
ElasticAccountIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<AccountState> schema) {
super(cfg, sitePaths, schema, clientBuilder, ACCOUNTS);
this.accountCache = accountCache;
@@ -89,19 +87,17 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@Override
public void replace(AccountState as) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(ACCOUNTS)
.addAction(insert(ACCOUNTS, as))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(ACCOUNTS, getId(as), INDEX);
bulk += toDoc(as);
String uri = getURI(ACCOUNTS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace account %s in index %s: %s",
as.getAccount().getId(), indexName, result.getErrorMessage()));
as.getAccount().getId(), indexName, statusCode));
}
}
@@ -112,8 +108,8 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
@Override
protected Builder addActions(Builder builder, Account.Id c) {
return builder.addAction(delete(ACCOUNTS, c));
protected String addActions(Account.Id c) {
return delete(ACCOUNTS, c);
}
@Override
@@ -128,7 +124,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
private class QuerySource implements DataSource<AccountState> {
private final Search search;
private final String search;
private final Set<String> fields;
QuerySource(Predicate<AccountState> p, QueryOptions opts) throws QueryParseException {
@@ -141,15 +137,8 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(ACCOUNTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
JsonArray sortArray = getSortArray(AccountField.ID.getName());
search = getSearch(searchSource, sortArray);
}
@Override
@@ -161,9 +150,14 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
public ResultSet<AccountState> read() throws OrmException {
try {
List<AccountState> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(ACCOUNTS, SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
@@ -172,7 +166,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<AccountState> r = Collections.unmodifiableList(results);
return new ResultSet<AccountState>() {
@@ -196,11 +190,6 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
}
}
@Override
public String toString() {
return search.toString();
}
private AccountState toAccountState(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {

96
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@@ -24,7 +24,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
@@ -32,6 +31,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.Predicate;
@@ -55,26 +56,24 @@ import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,7 +107,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
SitePaths sitePaths,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<ChangeData> schema) {
super(cfg, sitePaths, schema, clientBuilder, CHANGES);
this.db = db;
@@ -133,20 +132,17 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
throw new IOException(e);
}
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(CHANGES)
.addAction(insert(insertIndex, cd))
.addAction(delete(deleteIndex, cd.getId()))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(insertIndex, getId(cd), INDEX);
bulk += toDoc(cd);
bulk += toAction(deleteIndex, cd.getId().toString(), DELETE);
String uri = getURI(CHANGES, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace change %s in index %s: %s",
cd.getId(), indexName, result.getErrorMessage()));
"Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode));
}
}
@@ -165,8 +161,8 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
@Override
protected Builder addActions(Builder builder, Id c) {
return builder.addAction(delete(OPEN_CHANGES, c)).addAction(delete(OPEN_CHANGES, c));
protected String addActions(Id c) {
return delete(OPEN_CHANGES, c) + delete(CLOSED_CHANGES, c);
}
@Override
@@ -180,18 +176,12 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
private class QuerySource implements ChangeDataSource {
private final Search search;
private final String search;
private final Set<String> fields;
private final List<String> types;
QuerySource(List<String> types, Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
List<Sort> sorts =
ImmutableList.of(
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
for (Sort sort : sorts) {
sort.setIgnoreUnmapped();
}
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.changeFields(opts);
SearchSourceBuilder searchSource =
@@ -201,12 +191,8 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
.size(opts.limit())
.fields(Lists.newArrayList(fields));
search =
new Search.Builder(searchSource.toString())
.addType(types)
.addSort(sorts)
.addIndex(indexName)
.build();
search = getSearch(searchSource, getSortArray());
this.types = types;
}
@Override
@@ -218,9 +204,14 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
public ResultSet<ChangeData> read() throws OrmException {
try {
List<ChangeData> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(types);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
@@ -229,7 +220,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<ChangeData> r = Collections.unmodifiableList(results);
return new ResultSet<ChangeData>() {
@@ -258,11 +249,6 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
return false;
}
@Override
public String toString() {
return search.toString();
}
private ChangeData toChangeData(JsonElement json) {
JsonElement sourceElement = json.getAsJsonObject().get("_source");
if (sourceElement == null) {
@@ -438,5 +424,21 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
out.setUnresolvedCommentCount(count.getAsInt());
}
private JsonArray getSortArray() {
JsonObject properties = new JsonObject();
properties.addProperty(ORDER, "desc");
properties.addProperty(IGNORE_UNMAPPED, true);
JsonArray sortArray = new JsonArray();
addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
addNamedElement(ChangeField.LEGACY_ID.getName(), properties, sortArray);
return sortArray;
}
}
private String getURI(List<String> types) throws UnsupportedEncodingException {
String joinedTypes = String.join(",", types);
return getURI(joinedTypes, SEARCH);
}
}

30
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
View File

@@ -18,13 +18,12 @@ import com.google.common.base.MoreObjects;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpHost;
import org.eclipse.jgit.lib.Config;
@Singleton
@@ -33,7 +32,7 @@ class ElasticConfiguration {
private static final String DEFAULT_PORT = "9200";
private static final String DEFAULT_PROTOCOL = "http";
final List<String> urls;
final List<HttpHost> urls;
final String username;
final String password;
final boolean requestCompression;
@@ -59,14 +58,18 @@ class ElasticConfiguration {
Set<String> subsections = cfg.getSubsections("elasticsearch");
if (subsections.isEmpty()) {
this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT));
HttpHost httpHost =
new HttpHost(DEFAULT_HOST, Integer.valueOf(DEFAULT_PORT), DEFAULT_PROTOCOL);
this.urls = Collections.singletonList(httpHost);
} else {
this.urls = new ArrayList<>(subsections.size());
for (String subsection : subsections) {
String port = getString(cfg, subsection, "port", DEFAULT_PORT);
String host = getString(cfg, subsection, "hostname", DEFAULT_HOST);
String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL);
this.urls.add(buildUrl(protocol, host, port));
HttpHost httpHost = new HttpHost(host, Integer.valueOf(port), protocol);
this.urls.add(httpHost);
}
}
}
@@ -74,19 +77,4 @@ class ElasticConfiguration {
private String getString(Config cfg, String subsection, String name, String defaultValue) {
return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
}
private String buildUrl(String protocol, String hostname, String port) {
try {
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
} catch (MalformedURLException | NumberFormatException e) {
throw new RuntimeException(
"Cannot build url to Elasticsearch from values: protocol="
+ protocol
+ " hostname="
+ hostname
+ " port="
+ port,
e);
}
}
}

82
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@@ -14,10 +14,11 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
@@ -34,26 +35,23 @@ import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.client.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,19 +65,19 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
}
static final String GROUPS = "groups";
public static final String GROUPS = "groups";
private static final Logger log = LoggerFactory.getLogger(ElasticGroupIndex.class);
private final GroupMapping mapping;
private final Provider<GroupCache> groupCache;
@Inject
@AssistedInject
ElasticGroupIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
JestClientBuilder clientBuilder,
ElasticRestClientBuilder clientBuilder,
@Assisted Schema<InternalGroup> schema) {
super(cfg, sitePaths, schema, clientBuilder, GROUPS);
this.groupCache = groupCache;
@@ -88,19 +86,17 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@Override
public void replace(InternalGroup group) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(GROUPS)
.addAction(insert(GROUPS, group))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
String bulk = toAction(GROUPS, getId(group), INDEX);
bulk += toDoc(group);
String uri = getURI(GROUPS, BULK);
Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
String.format(
"Failed to replace group %s in index %s: %s",
group.getGroupUUID().get(), indexName, result.getErrorMessage()));
group.getGroupUUID().get(), indexName, statusCode));
}
}
@@ -111,8 +107,8 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
@Override
protected Builder addActions(Builder builder, AccountGroup.UUID c) {
return builder.addAction(delete(GROUPS, c));
protected String addActions(AccountGroup.UUID c) {
return delete(GROUPS, c);
}
@Override
@@ -127,7 +123,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
private class QuerySource implements DataSource<InternalGroup> {
private final Search search;
private final String search;
private final Set<String> fields;
QuerySource(Predicate<InternalGroup> p, QueryOptions opts) throws QueryParseException {
@@ -140,15 +136,8 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(GroupField.UUID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(GROUPS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
JsonArray sortArray = getSortArray(GroupField.UUID.getName());
search = getSearch(searchSource, sortArray);
}
@Override
@@ -160,19 +149,23 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
public ResultSet<InternalGroup> read() throws OrmException {
try {
List<InternalGroup> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
String uri = getURI(GROUPS, SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
String content = getContent(response);
JsonObject obj =
new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
Optional<InternalGroup> internalGroup = toInternalGroup(json.get(i));
internalGroup.ifPresent(results::add);
results.add(toAccountGroup(json.get(i)).get());
}
}
} else {
log.error(result.getErrorMessage());
log.error(statusLine.getReasonPhrase());
}
final List<InternalGroup> r = Collections.unmodifiableList(results);
return new ResultSet<InternalGroup>() {
@@ -196,12 +189,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
}
}
@Override
public String toString() {
return search.toString();
}
private Optional<InternalGroup> toInternalGroup(JsonElement json) {
private Optional<InternalGroup> toAccountGroup(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");

22
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
View File

@@ -16,31 +16,37 @@ package com.google.gerrit.elasticsearch;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.indices.aliases.GetAliases;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
@Singleton
class ElasticIndexVersionDiscovery {
private final JestHttpClient client;
private final RestClient client;
@Inject
ElasticIndexVersionDiscovery(JestClientBuilder clientBuilder) {
ElasticIndexVersionDiscovery(ElasticRestClientBuilder clientBuilder) {
this.client = clientBuilder.build();
}
List<String> discover(String prefix, String indexName) throws IOException {
String name = prefix + indexName + "_";
JestResult result = client.execute(new GetAliases.Builder().addIndex(name + "*").build());
if (result.isSucceeded()) {
JsonObject object = result.getJsonObject().getAsJsonObject();
Response response = client.performRequest(HttpGet.METHOD_NAME, name + "*/_aliases");
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
String content = AbstractElasticIndex.getContent(response);
JsonObject object = new JsonParser().parse(content).getAsJsonObject();
List<String> versions = new ArrayList<>(object.size());
for (Entry<String, JsonElement> entry : object.entrySet()) {
versions.add(entry.getKey().replace(name, ""));

31
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
View File

@@ -14,6 +14,9 @@
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.builders.BoolQueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilders;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.query.AndPredicate;
@@ -28,10 +31,6 @@ import com.google.gerrit.index.query.RegexPredicate;
import com.google.gerrit.index.query.TimestampRangePredicate;
import com.google.gerrit.server.query.change.AfterPredicate;
import java.time.Instant;
import org.apache.lucene.search.BooleanQuery;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
public class ElasticQueryBuilder {
@@ -52,27 +51,19 @@ public class ElasticQueryBuilder {
}
private <T> BoolQueryBuilder and(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
}
private <T> BoolQueryBuilder or(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
}
private <T> QueryBuilder not(Predicate<T> p) throws QueryParseException {

58
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticRestClientBuilder.java Normal file
View File

@@ -0,0 +1,58 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
@Singleton
class ElasticRestClientBuilder {
private final HttpHost[] hosts;
private final String username;
private final String password;
@Inject
ElasticRestClientBuilder(ElasticConfiguration cfg) {
hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]);
username = cfg.username;
password = cfg.password;
}
RestClient build() {
RestClientBuilder builder = RestClient.builder(hosts);
setConfiguredCredentialsIfAny(builder);
return builder.build();
}
private void setConfiguredCredentialsIfAny(RestClientBuilder builder) {
if (username != null && password != null) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(
(HttpAsyncClientBuilder httpClientBuilder) ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
}
}

54
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/JestClientBuilder.java
View File

@@ -1,54 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.client.config.HttpClientConfig.Builder;
import io.searchbox.client.http.JestHttpClient;
import java.util.concurrent.TimeUnit;
@Singleton
class JestClientBuilder {
private final ElasticConfiguration cfg;
@Inject
JestClientBuilder(ElasticConfiguration cfg) {
this.cfg = cfg;
}
JestHttpClient build() {
JestClientFactory factory = new JestClientFactory();
Builder builder =
new HttpClientConfig.Builder(cfg.urls)
.multiThreaded(true)
.discoveryEnabled(false)
.connTimeout((int) cfg.connectionTimeout)
.maxConnectionIdleTime(cfg.maxConnectionIdleTime, cfg.maxConnectionIdleUnit)
.maxTotalConnection(cfg.maxTotalConnection)
.readTimeout(cfg.readTimeout)
.requestCompressionEnabled(cfg.requestCompression)
.discoveryFrequency(1L, TimeUnit.MINUTES);
if (cfg.username != null && cfg.password != null) {
builder.defaultCredentials(cfg.username, cfg.password);
}
factory.setHttpClientConfig(builder.build());
return (JestHttpClient) factory.getObject();
}
}

88
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java Normal file
View File

@@ -0,0 +1,88 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* A Query that matches documents matching boolean combinations of other queries. A trimmed down
* version of {@link org.elasticsearch.index.query.BoolQueryBuilder} for this very package.
*/
public class BoolQueryBuilder extends QueryBuilder {
private final List<QueryBuilder> mustClauses = new ArrayList<>();
private final List<QueryBuilder> mustNotClauses = new ArrayList<>();
private final List<QueryBuilder> filterClauses = new ArrayList<>();
private final List<QueryBuilder> shouldClauses = new ArrayList<>();
/**
* Adds a query that <b>must</b> appear in the matching documents and will contribute to scoring.
*/
public BoolQueryBuilder must(QueryBuilder queryBuilder) {
mustClauses.add(queryBuilder);
return this;
}
/**
* Adds a query that <b>must not</b> appear in the matching documents and will not contribute to
* scoring.
*/
public BoolQueryBuilder mustNot(QueryBuilder queryBuilder) {
mustNotClauses.add(queryBuilder);
return this;
}
/**
* Adds a query that <i>should</i> appear in the matching documents. For a boolean query with no
* <tt>MUST</tt> clauses one or more <code>SHOULD</code> clauses must match a document for the
* BooleanQuery to match.
*/
public BoolQueryBuilder should(QueryBuilder queryBuilder) {
shouldClauses.add(queryBuilder);
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("bool");
doXArrayContent("must", mustClauses, builder);
doXArrayContent("filter", filterClauses, builder);
doXArrayContent("must_not", mustNotClauses, builder);
doXArrayContent("should", shouldClauses, builder);
builder.endObject();
}
private void doXArrayContent(String field, List<QueryBuilder> clauses, XContentBuilder builder)
throws IOException {
if (clauses.isEmpty()) {
return;
}
if (clauses.size() == 1) {
builder.field(field);
clauses.get(0).toXContent(builder);
} else {
builder.startArray(field);
for (QueryBuilder clause : clauses) {
clause.toXContent(builder);
}
builder.endArray();
}
}
}

37
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java Normal file
View File

@@ -0,0 +1,37 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* Constructs a query that only match on documents that the field has a value in them. A trimmed
* down version of {@link org.elasticsearch.index.query.ExistsQueryBuilder} for this very package.
*/
class ExistsQueryBuilder extends QueryBuilder {
private final String name;
ExistsQueryBuilder(String name) {
this.name = name;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("exists");
builder.field("field", name);
builder.endObject();
}
}

30
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java Normal file
View File

@@ -0,0 +1,30 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A query that matches on all documents. A trimmed down version of {@link
* org.elasticsearch.index.query.MatchAllQueryBuilder} for this very package.
*/
class MatchAllQueryBuilder extends QueryBuilder {
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("match_all");
builder.endObject();
}
}

64
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java Normal file
View File

@@ -0,0 +1,64 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.Locale;
/**
* Match query is a query that analyzes the text and constructs a query as the result of the
* analysis. It can construct different queries based on the type provided. A trimmed down version
* of {@link org.elasticsearch.index.query.MatchQueryBuilder} for this very package.
*/
class MatchQueryBuilder extends QueryBuilder {
enum Type {
/** The text is analyzed and used as a phrase query. */
PHRASE,
/** The text is analyzed and used in a phrase query, with the last term acting as a prefix. */
PHRASE_PREFIX
}
private final String name;
private final Object text;
private Type type;
/** Constructs a new text query. */
MatchQueryBuilder(String name, Object text) {
this.name = name;
this.text = text;
}
/** Sets the type of the text query. */
MatchQueryBuilder type(Type type) {
this.type = type;
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("match");
builder.startObject(name);
builder.field("query", text);
if (type != null) {
builder.field("type", type.toString().toLowerCase(Locale.ENGLISH));
}
builder.endObject();
builder.endObject();
}
}

34
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java Normal file
View File

@@ -0,0 +1,34 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down version of {@link org.elasticsearch.index.query.QueryBuilder} for this very
* package.
*/
public abstract class QueryBuilder {
protected QueryBuilder() {}
protected void toXContent(XContentBuilder builder) throws IOException {
builder.startObject();
doXContent(builder);
builder.endObject();
}
protected abstract void doXContent(XContentBuilder builder) throws IOException;
}

102
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java Normal file
View File

@@ -0,0 +1,102 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
/**
* A static factory for simple "import static" usage. A trimmed down version of {@link
* org.elasticsearch.index.query.QueryBuilders} for this very package.
*/
public abstract class QueryBuilders {
/** A query that match on all documents. */
public static MatchAllQueryBuilder matchAllQuery() {
return new MatchAllQueryBuilder();
}
/**
* Creates a text query with type "PHRASE" for the provided field name and text.
*
* @param name The field name.
* @param text The query text (to be analyzed).
*/
public static MatchQueryBuilder matchPhraseQuery(String name, Object text) {
return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE);
}
/**
* Creates a match query with type "PHRASE_PREFIX" for the provided field name and text.
*
* @param name The field name.
* @param text The query text (to be analyzed).
*/
public static MatchQueryBuilder matchPhrasePrefixQuery(String name, Object text) {
return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE_PREFIX);
}
/**
* A Query that matches documents containing a term.
*
* @param name The name of the field
* @param value The value of the term
*/
public static TermQueryBuilder termQuery(String name, String value) {
return new TermQueryBuilder(name, value);
}
/**
* A Query that matches documents containing a term.
*
* @param name The name of the field
* @param value The value of the term
*/
public static TermQueryBuilder termQuery(String name, int value) {
return new TermQueryBuilder(name, value);
}
/**
* A Query that matches documents within an range of terms.
*
* @param name The field name
*/
public static RangeQueryBuilder rangeQuery(String name) {
return new RangeQueryBuilder(name);
}
/**
* A Query that matches documents containing terms with a specified regular expression.
*
* @param name The name of the field
* @param regexp The regular expression
*/
public static RegexpQueryBuilder regexpQuery(String name, String regexp) {
return new RegexpQueryBuilder(name, regexp);
}
/** A Query that matches documents matching boolean combinations of other queries. */
public static BoolQueryBuilder boolQuery() {
return new BoolQueryBuilder();
}
/**
* A filter to filter only documents where a field exists in them.
*
* @param name The name of the field
*/
public static ExistsQueryBuilder existsQuery(String name) {
return new ExistsQueryBuilder(name);
}
private QueryBuilders() {}
}

35
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java Normal file
View File

@@ -0,0 +1,35 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down and further altered version of {@link
* org.elasticsearch.action.support.QuerySourceBuilder} for this very package.
*/
class QuerySourceBuilder {
private final QueryBuilder queryBuilder;
QuerySourceBuilder(QueryBuilder queryBuilder) {
this.queryBuilder = queryBuilder;
}
void innerToXContent(XContentBuilder builder) throws IOException {
builder.field("query");
queryBuilder.toXContent(builder);
}
}

88
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java Normal file
View File

@@ -0,0 +1,88 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents within an range of terms. A trimmed down version of {@link
* org.elasticsearch.index.query.RangeQueryBuilder} for this very package.
*/
public class RangeQueryBuilder extends QueryBuilder {
private final String name;
private Object from;
private Object to;
private boolean includeLower = true;
private boolean includeUpper = true;
/**
* A Query that matches documents within an range of terms.
*
* @param name The field name
*/
RangeQueryBuilder(String name) {
this.name = name;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gt(Object from) {
this.from = from;
this.includeLower = false;
return this;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gte(Object from) {
this.from = from;
this.includeLower = true;
return this;
}
/** The from part of the range query. Null indicates unbounded. */
public RangeQueryBuilder gte(int from) {
this.from = from;
this.includeLower = true;
return this;
}
/** The to part of the range query. Null indicates unbounded. */
public RangeQueryBuilder lte(Object to) {
this.to = to;
this.includeUpper = true;
return this;
}
/** The to part of the range query. Null indicates unbounded. */
public RangeQueryBuilder lte(int to) {
this.to = to;
this.includeUpper = true;
return this;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("range");
builder.startObject(name);
builder.field("from", from);
builder.field("to", to);
builder.field("include_lower", includeLower);
builder.field("include_upper", includeUpper);
builder.endObject();
builder.endObject();
}
}

50
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that does fuzzy matching for a specific value. A trimmed down version of {@link
* org.elasticsearch.index.query.RegexpQueryBuilder} for this very package.
*/
class RegexpQueryBuilder extends QueryBuilder {
private final String name;
private final String regexp;
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param regexp The regular expression
*/
RegexpQueryBuilder(String name, String regexp) {
this.name = name;
this.regexp = regexp;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("regexp");
builder.startObject(name);
builder.field("value", regexp);
builder.field("flags_value", 65535);
builder.endObject();
builder.endObject();
}
}

108
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java Normal file
View File

@@ -0,0 +1,108 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
import java.util.List;
/**
* A search source builder allowing to easily build search source. A trimmed down and further
* altered version of {@link org.elasticsearch.search.builder.SearchSourceBuilder} for this very
* package.
*/
public class SearchSourceBuilder {
private QuerySourceBuilder querySourceBuilder;
private int from = -1;
private int size = -1;
private List<String> fieldNames;
/** Constructs a new search source builder. */
public SearchSourceBuilder() {}
/** Constructs a new search source builder with a search query. */
public SearchSourceBuilder query(QueryBuilder query) {
if (this.querySourceBuilder == null) {
this.querySourceBuilder = new QuerySourceBuilder(query);
}
return this;
}
/** From index to start the search from. Defaults to <tt>0</tt>. */
public SearchSourceBuilder from(int from) {
this.from = from;
return this;
}
/** The number of search hits to return. Defaults to <tt>10</tt>. */
public SearchSourceBuilder size(int size) {
this.size = size;
return this;
}
/**
* Sets the fields to load and return as part of the search request. If none are specified, the
* source of the document will be returned.
*/
public SearchSourceBuilder fields(List<String> fields) {
this.fieldNames = fields;
return this;
}
@Override
public final String toString() {
try {
XContentBuilder builder = new XContentBuilder();
toXContent(builder);
return builder.string();
} catch (IOException ioe) {
return "";
}
}
private void toXContent(XContentBuilder builder) throws IOException {
builder.startObject();
innerToXContent(builder);
builder.endObject();
}
private void innerToXContent(XContentBuilder builder) throws IOException {
if (from != -1) {
builder.field("from", from);
}
if (size != -1) {
builder.field("size", size);
}
if (querySourceBuilder != null) {
querySourceBuilder.innerToXContent(builder);
}
if (fieldNames != null) {
if (fieldNames.size() == 1) {
builder.field("fields", fieldNames.get(0));
} else {
builder.startArray("fields");
for (String fieldName : fieldNames) {
builder.value(fieldName);
}
builder.endArray();
}
}
}
}

66
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java Normal file
View File

@@ -0,0 +1,66 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents containing a term. A trimmed down version of {@link
* org.elasticsearch.index.query.TermQueryBuilder} for this very package.
*/
class TermQueryBuilder extends QueryBuilder {
private final String name;
private final Object value;
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
TermQueryBuilder(String name, String value) {
this(name, (Object) value);
}
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
TermQueryBuilder(String name, int value) {
this(name, (Object) value);
}
/**
* Constructs a new term query.
*
* @param name The name of the field
* @param value The value of the term
*/
private TermQueryBuilder(String name, Object value) {
this.name = name;
this.value = value;
}
@Override
protected void doXContent(XContentBuilder builder) throws IOException {
builder.startObject("term");
builder.field(name, value);
builder.endObject();
}
}

170
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java Normal file
View File

@@ -0,0 +1,170 @@
// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch.builders;
import static java.time.format.DateTimeFormatter.ISO_INSTANT;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.google.common.base.Charsets;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.util.Date;
/**
* A trimmed down and further altered version of {@link
* org.elasticsearch.common.xcontent.XContentBuilder} for this very package.
*/
public final class XContentBuilder implements Closeable {
private final JsonGenerator generator;
private final ByteArrayOutputStream bos = new ByteArrayOutputStream();
/**
* Constructs a new builder. Make sure to call {@link #close()} when the builder is done with.
* Inspired from {@link org.elasticsearch.common.xcontent.json.JsonXContent} static block.
*/
public XContentBuilder() throws IOException {
JsonFactory jsonFactory = new JsonFactory();
jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
jsonFactory.configure(
JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW,
false); // this trips on many mappings now...
this.generator = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
}
public XContentBuilder startObject(String name) throws IOException {
field(name);
startObject();
return this;
}
public XContentBuilder startObject() throws IOException {
generator.writeStartObject();
return this;
}
public XContentBuilder endObject() throws IOException {
generator.writeEndObject();
return this;
}
public void startArray(String name) throws IOException {
field(name);
startArray();
}
private void startArray() throws IOException {
generator.writeStartArray();
}
public void endArray() throws IOException {
generator.writeEndArray();
}
public XContentBuilder field(String name) throws IOException {
generator.writeFieldName(name);
return this;
}
public XContentBuilder field(String name, String value) throws IOException {
field(name);
generator.writeString(value);
return this;
}
public XContentBuilder field(String name, int value) throws IOException {
field(name);
generator.writeNumber(value);
return this;
}
public XContentBuilder field(String name, Iterable<?> value) throws IOException {
startArray(name);
for (Object o : value) {
value(o);
}
endArray();
return this;
}
public XContentBuilder field(String name, Object value) throws IOException {
field(name);
writeValue(value);
return this;
}
public XContentBuilder value(Object value) throws IOException {
writeValue(value);
return this;
}
public XContentBuilder field(String name, boolean value) throws IOException {
field(name);
generator.writeBoolean(value);
return this;
}
public XContentBuilder value(String value) throws IOException {
generator.writeString(value);
return this;
}
@Override
public void close() {
try {
generator.close();
} catch (IOException e) {
// ignore
}
}
/** Returns a string representation of the builder (only applicable for text based xcontent). */
public String string() {
close();
byte[] bytesArray = bos.toByteArray();
return new String(bytesArray, Charsets.UTF_8);
}
private void writeValue(Object value) throws IOException {
if (value == null) {
generator.writeNull();
return;
}
Class<?> type = value.getClass();
if (type == String.class) {
generator.writeString((String) value);
} else if (type == Integer.class) {
generator.writeNumber(((Integer) value));
} else if (type == byte[].class) {
generator.writeBinary((byte[]) value);
} else if (value instanceof Date) {
generator.writeString(ISO_INSTANT.format(((Date) value).toInstant()));
} else {
// if this is a "value" object, like enum, DistanceUnit, ..., just toString it
// yea, it can be misleading when toString a Java class, but really, jackson should be used in
// that case
generator.writeString(value.toString());
// throw new ElasticsearchIllegalArgumentException("type not supported for generic value
// conversion: " + type);
}
}
}

81
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java Normal file
View File

@@ -0,0 +1,81 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.apache.http.HttpHost;
import org.junit.internal.AssumptionViolatedException;
import org.testcontainers.containers.GenericContainer;
/* Helper class for running ES integration tests in docker container */
public class ElasticContainer<SELF extends ElasticContainer<SELF>> extends GenericContainer<SELF> {
private static final int ELASTICSEARCH_DEFAULT_PORT = 9200;
public enum Version {
V2,
V5,
V6
}
public static ElasticContainer<?> createAndStart(Version version) {
// Assumption violation is not natively supported by Testcontainers.
// See https://github.com/testcontainers/testcontainers-java/issues/343
try {
ElasticContainer<?> container = new ElasticContainer<>(version);
container.start();
return container;
} catch (Throwable t) {
throw new AssumptionViolatedException("Unable to start container", t);
}
}
public static ElasticContainer<?> createAndStart() {
return createAndStart(Version.V2);
}
private static String getImageName(Version version) {
switch (version) {
case V2:
return "elasticsearch:2.4.6-alpine";
case V5:
return "elasticsearch:5.6.9-alpine";
case V6:
return "docker.elastic.co/elasticsearch/elasticsearch:6.2.4";
}
throw new IllegalStateException("Unsupported version: " + version.name());
}
private ElasticContainer(Version version) {
super(getImageName(version));
}
@Override
protected void configure() {
addExposedPort(ELASTICSEARCH_DEFAULT_PORT);
// https://github.com/docker-library/elasticsearch/issues/58
addEnv("-Ees.network.host", "0.0.0.0");
}
@Override
protected Set<Integer> getLivenessCheckPorts() {
return ImmutableSet.of(getMappedPort(ELASTICSEARCH_DEFAULT_PORT));
}
public HttpHost getHttpHost() {
return new HttpHost(getContainerIpAddress(), getMappedPort(ELASTICSEARCH_DEFAULT_PORT));
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryAccountsTest.java
View File

@@ -19,29 +19,29 @@ import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryAccountsTest extends AbstractQueryAccountsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryChangesTest.java
View File

@@ -20,7 +20,6 @@ import com.google.gerrit.testutil.InMemoryModule;
import com.google.gerrit.testutil.InMemoryRepositoryManager.Repo;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
@@ -29,22 +28,23 @@ import org.junit.Test;
public class ElasticQueryChangesTest extends AbstractQueryChangesTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryGroupsTest.java
View File

@@ -19,29 +19,29 @@ import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryGroupsTest extends AbstractQueryGroupsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

91
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
View File

@@ -14,90 +14,31 @@
package com.google.gerrit.elasticsearch;
import static com.google.common.truth.Truth.assertThat;
import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
public final class ElasticTestUtils {
public static class ElasticNodeInfo {
public final Node node;
public final String port;
public final File elasticDir;
public final int port;
private ElasticNodeInfo(Node node, File rootDir, String port) {
this.node = node;
public ElasticNodeInfo(int port) {
this.port = port;
this.elasticDir = rootDir;
}
}
static void configure(Config config, String port, String prefix) {
public static void configure(Config config, int port, String prefix) {
config.setEnum("index", null, "type", IndexType.ELASTICSEARCH);
config.setString("elasticsearch", "test", "protocol", "http");
config.setString("elasticsearch", "test", "hostname", "localhost");
config.setString("elasticsearch", "test", "port", port);
config.setInt("elasticsearch", "test", "port", port);
config.setString("elasticsearch", null, "prefix", prefix);
}
static ElasticNodeInfo startElasticsearchNode() throws InterruptedException, ExecutionException {
File elasticDir = Files.createTempDir();
Path elasticDirPath = elasticDir.toPath();
Settings settings =
Settings.settingsBuilder()
.put("cluster.name", "gerrit")
.put("node.name", "Gerrit Elasticsearch Test Node")
.put("node.local", true)
.put("discovery.zen.ping.multicast.enabled", false)
.put("index.store.fs.memory.enabled", true)
.put("index.gateway.type", "none")
.put("index.max_result_window", Integer.MAX_VALUE)
.put("gateway.type", "default")
.put("http.port", 0)
.put("discovery.zen.ping.unicast.hosts", "[\"localhost\"]")
.put("path.home", elasticDirPath.toAbsolutePath())
.put("path.data", elasticDirPath.resolve("data").toAbsolutePath())
.put("path.work", elasticDirPath.resolve("work").toAbsolutePath())
.put("path.logs", elasticDirPath.resolve("logs").toAbsolutePath())
.put("transport.tcp.connect_timeout", "60s")
.build();
// Start the node
Node node = NodeBuilder.nodeBuilder().settings(settings).node();
// Wait for it to be ready
node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
assertThat(node.isClosed()).isFalse();
return new ElasticNodeInfo(node, elasticDir, getHttpPort(node));
}
static class NodeInfo {
String httpAddress;
}
static class Info {
Map<String, NodeInfo> nodes;
config.setInt("index", null, "maxLimit", 10000);
}
public static void createAllIndexes(Injector injector) throws IOException {
@@ -108,28 +49,6 @@ public final class ElasticTestUtils {
}
}
private static String getHttpPort(Node node) throws InterruptedException, ExecutionException {
String nodes =
node.client().admin().cluster().nodesInfo(new NodesInfoRequest("*")).get().toString();
Gson gson =
new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();
Info info = gson.fromJson(nodes, Info.class);
if (info.nodes == null || info.nodes.size() != 1) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
Iterator<NodeInfo> values = info.nodes.values().iterator();
String httpAddress = values.next().httpAddress;
if (Strings.isNullOrEmpty(httpAddress)) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
if (httpAddress.indexOf(':') < 0) {
throw new RuntimeException("Seems that port is not included in Elasticsearch http_address");
}
return httpAddress.substring(httpAddress.indexOf(':') + 1, httpAddress.length());
}
private ElasticTestUtils() {
// hide default constructor
}