diff --git a/WORKSPACE b/WORKSPACE index 757d86e75c..bd084f60bd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -918,21 +918,6 @@ maven_jar( sha1 = "d2954e1173a608a9711f132d1768a676a8b1fb81", ) -# Java REST client for Elasticsearch. -JEST_VERSION = "2.4.0" - -maven_jar( - name = "jest_common", - artifact = "io.searchbox:jest-common:" + JEST_VERSION, - sha1 = "ea779ebe7c438a53dce431f85b0d4e1d8faee2ac", -) - -maven_jar( - name = "jest", - artifact = "io.searchbox:jest:" + JEST_VERSION, - sha1 = "e2a604a584e6633545ac6b1fe99ef888ab96dae9", -) - maven_jar( name = "joda_time", artifact = "joda-time:joda-time:2.9.9", @@ -945,6 +930,12 @@ maven_jar( sha1 = "675642ac208e0b741bc9118dcbcae44c271b992a", ) +maven_jar( + name = "elasticsearch-rest-client", + artifact = "org.elasticsearch.client:elasticsearch-rest-client:5.6.9", + sha1 = "895706412e2fba3f842fca82ec3dece1cb4ee7d1", +) + maven_jar( name = "compress_lzf", artifact = "com.ning:compress-lzf:1.0.2", diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index 107fb0f499..dd3c320037 100644 --- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java @@ -16,18 +16,21 @@ package com.google.gerrit.elasticsearch; import static com.google.common.base.Preconditions.checkArgument; import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.toList; import static org.apache.commons.codec.binary.Base64.decodeBase64; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Streams; +import com.google.common.io.CharStreams; +import com.google.gerrit.elasticsearch.builders.QueryBuilder; +import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder; +import com.google.gerrit.elasticsearch.builders.XContentBuilder; import com.google.gerrit.index.FieldDef; import com.google.gerrit.index.FieldType; import com.google.gerrit.index.Index; @@ -46,39 +49,50 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.gwtorm.protobuf.ProtobufCodec; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.ResultSet; -import io.searchbox.client.JestResult; -import io.searchbox.client.http.JestHttpClient; -import io.searchbox.core.Bulk; -import io.searchbox.core.Delete; -import io.searchbox.core.Search; -import io.searchbox.core.search.sort.Sort; -import io.searchbox.indices.CreateIndex; -import io.searchbox.indices.DeleteIndex; -import io.searchbox.indices.IndicesExists; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.sql.Timestamp; -import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.function.Function; import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.ContentType; +import org.apache.http.nio.entity.NStringEntity; import org.eclipse.jgit.lib.Config; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; abstract class AbstractElasticIndex implements Index { private static final Logger log = LoggerFactory.getLogger(AbstractElasticIndex.class); + protected static final String BULK = "_bulk"; + protected static final String DELETE = "delete"; + protected static final String IGNORE_UNMAPPED = "ignore_unmapped"; + protected static final String INDEX = "index"; + protected static final String ORDER = "order"; + protected static final String SEARCH = "_search"; + protected static List decodeProtos( JsonObject doc, String fieldName, ProtobufCodec codec) { JsonArray field = doc.getAsJsonArray(fieldName); @@ -90,12 +104,24 @@ abstract class AbstractElasticIndex implements Index { .toList(); } + static String getContent(Response response) throws IOException { + HttpEntity responseEntity = response.getEntity(); + String content = ""; + if (responseEntity != null) { + InputStream contentStream = responseEntity.getContent(); + try (Reader reader = new InputStreamReader(contentStream)) { + content = CharStreams.toString(reader); + } + } + return content; + } + private final Schema schema; private final SitePaths sitePaths; private final String indexNameRaw; + private final RestClient client; protected final String indexName; - protected final JestHttpClient client; protected final Gson gson; protected final ElasticQueryBuilder queryBuilder; @@ -103,7 +129,7 @@ abstract class AbstractElasticIndex implements Index { @GerritServerConfig Config cfg, SitePaths sitePaths, Schema schema, - JestClientBuilder clientBuilder, + ElasticRestClientBuilder clientBuilder, String indexName) { this.sitePaths = sitePaths; this.schema = schema; @@ -126,7 +152,11 @@ abstract class AbstractElasticIndex implements Index { @Override public void close() { - client.shutdownClient(); + try { + client.close(); + } catch (IOException e) { + // Ignored. + } } @Override @@ -136,60 +166,57 @@ abstract class AbstractElasticIndex implements Index { @Override public void delete(K c) throws IOException { - Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build(); - JestResult result = client.execute(bulk); - if (!result.isSucceeded()) { + String uri = getURI(indexNameRaw, BULK); + Response response = performRequest(HttpPost.METHOD_NAME, addActions(c), uri, getRefreshParam()); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { throw new IOException( - String.format( - "Failed to delete change %s in index %s: %s", - c, indexName, result.getErrorMessage())); + String.format("Failed to delete change %s in index %s: %s", c, indexName, statusCode)); } } @Override public void deleteAll() throws IOException { // Delete the index, if it exists. - JestResult result = client.execute(new IndicesExists.Builder(indexName).build()); - if (result.isSucceeded()) { - result = client.execute(new DeleteIndex.Builder(indexName).build()); - if (!result.isSucceeded()) { + Response response = client.performRequest(HttpHead.METHOD_NAME, indexName); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_OK) { + response = client.performRequest(HttpDelete.METHOD_NAME, indexName); + statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { throw new IOException( - String.format("Failed to delete index %s: %s", indexName, result.getErrorMessage())); + String.format("Failed to delete index %s: %s", indexName, statusCode)); } } // Recreate the index. - result = client.execute(new CreateIndex.Builder(indexName).settings(getMappings()).build()); - if (!result.isSucceeded()) { - String error = - String.format("Failed to create index %s: %s", indexName, result.getErrorMessage()); + response = + performRequest(HttpPut.METHOD_NAME, getMappings(), indexName, Collections.emptyMap()); + statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + String error = String.format("Failed to create index %s: %s", indexName, statusCode); throw new IOException(error); } } - protected abstract Bulk.Builder addActions(Bulk.Builder builder, K c); + protected abstract String addActions(K c); protected abstract String getMappings(); protected abstract String getId(V v); - protected Delete delete(String type, K c) { + protected String delete(String type, K c) { String id = c.toString(); - return new Delete.Builder(id).index(indexName).type(type).build(); - } - - protected io.searchbox.core.Index insert(String type, V v) throws IOException { - String id = getId(v); - String doc = toDocument(v); - return new io.searchbox.core.Index.Builder(doc).index(indexName).type(type).id(id).build(); + return toAction(type, id, DELETE); } private static boolean shouldAddElement(Object element) { return !(element instanceof String) || !((String) element).isEmpty(); } - private String toDocument(V v) throws IOException { - try (XContentBuilder builder = jsonBuilder().startObject()) { + protected String toDoc(V v) throws IOException { + try (XContentBuilder closeable = new XContentBuilder()) { + XContentBuilder builder = closeable.startObject(); for (Values values : schema.buildFields(v)) { String name = values.getField().getName(); if (values.getField().isRepeatable()) { @@ -205,7 +232,7 @@ abstract class AbstractElasticIndex implements Index { } } } - return builder.endObject().string(); + return builder.endObject().string() + System.lineSeparator(); } } @@ -214,16 +241,14 @@ abstract class AbstractElasticIndex implements Index { protected FieldBundle toFieldBundle(JsonObject doc) { Map> allFields = getSchema().getFields(); ListMultimap rawFields = ArrayListMultimap.create(); - for (Entry element : doc.get("fields").getAsJsonObject().entrySet()) { + for (Map.Entry element : doc.get("fields").getAsJsonObject().entrySet()) { checkArgument( allFields.containsKey(element.getKey()), "Unrecognized field " + element.getKey()); FieldType type = allFields.get(element.getKey()).getType(); - Iterable innerItems = element.getValue().isJsonArray() ? element.getValue().getAsJsonArray() : Collections.singleton(element.getValue()); - for (JsonElement inner : innerItems) { if (type == FieldType.EXACT || type == FieldType.FULL_TEXT || type == FieldType.PREFIX) { rawFields.put(element.getKey(), inner.getAsString()); @@ -243,19 +268,66 @@ abstract class AbstractElasticIndex implements Index { return new FieldBundle(rawFields); } + protected String toAction(String type, String id, String action) { + JsonObject properties = new JsonObject(); + properties.addProperty("_id", id); + properties.addProperty("_index", indexName); + properties.addProperty("_type", type); + + JsonObject jsonAction = new JsonObject(); + jsonAction.add(action, properties); + return jsonAction.toString() + System.lineSeparator(); + } + + protected void addNamedElement(String name, JsonObject element, JsonArray array) { + JsonObject arrayElement = new JsonObject(); + arrayElement.add(name, element); + array.add(arrayElement); + } + + protected Map getRefreshParam() { + Map params = new HashMap<>(); + params.put("refresh", "true"); + return params; + } + + protected String getSearch(SearchSourceBuilder searchSource, JsonArray sortArray) { + JsonObject search = new JsonParser().parse(searchSource.toString()).getAsJsonObject(); + search.add("sort", sortArray); + return gson.toJson(search); + } + + protected JsonArray getSortArray(String idFieldName) { + JsonObject properties = new JsonObject(); + properties.addProperty(ORDER, "asc"); + properties.addProperty(IGNORE_UNMAPPED, true); + + JsonArray sortArray = new JsonArray(); + addNamedElement(idFieldName, properties, sortArray); + return sortArray; + } + + protected String getURI(String type, String request) throws UnsupportedEncodingException { + String encodedType = URLEncoder.encode(type, UTF_8.toString()); + String encodedIndexName = URLEncoder.encode(indexName, UTF_8.toString()); + return encodedIndexName + "/" + encodedType + "/" + request; + } + + protected Response performRequest( + String method, String payload, String uri, Map params) throws IOException { + HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON); + return client.performRequest(method, uri, params, entity); + } + protected class ElasticQuerySource implements DataSource { private final QueryOptions opts; - private final Search search; + private final String search; + private final String index; - ElasticQuerySource(Predicate p, QueryOptions opts, String type, Sort sort) - throws QueryParseException { - this(p, opts, ImmutableList.of(type), ImmutableList.of(sort)); - } - - ElasticQuerySource( - Predicate p, QueryOptions opts, Collection types, Collection sorts) + ElasticQuerySource(Predicate p, QueryOptions opts, String index, JsonArray sortArray) throws QueryParseException { this.opts = opts; + this.index = index; QueryBuilder qb = queryBuilder.toQueryBuilder(p); SearchSourceBuilder searchSource = new SearchSourceBuilder() @@ -263,13 +335,7 @@ abstract class AbstractElasticIndex implements Index { .from(opts.start()) .size(opts.limit()) .fields(Lists.newArrayList(opts.fields())); - - search = - new Search.Builder(searchSource.toString()) - .addType(types) - .addSort(sorts) - .addIndex(indexName) - .build(); + search = getSearch(searchSource, sortArray); } @Override @@ -287,17 +353,17 @@ abstract class AbstractElasticIndex implements Index { return readImpl(AbstractElasticIndex.this::toFieldBundle); } - @Override - public String toString() { - return search.toString(); - } - private ResultSet readImpl(Function mapper) throws OrmException { try { List results = Collections.emptyList(); - JestResult result = client.execute(search); - if (result.isSucceeded()) { - JsonObject obj = result.getJsonObject().getAsJsonObject("hits"); + String uri = getURI(index, SEARCH); + Response response = + performRequest(HttpPost.METHOD_NAME, search, uri, Collections.emptyMap()); + StatusLine statusLine = response.getStatusLine(); + if (statusLine.getStatusCode() == HttpStatus.SC_OK) { + String content = getContent(response); + JsonObject obj = + new JsonParser().parse(content).getAsJsonObject().getAsJsonObject("hits"); if (obj.get("hits") != null) { JsonArray json = obj.getAsJsonArray("hits"); results = Lists.newArrayListWithCapacity(json.size()); @@ -309,7 +375,7 @@ abstract class AbstractElasticIndex implements Index { } } } else { - log.error(result.getErrorMessage()); + log.error(statusLine.getReasonPhrase()); } final List r = Collections.unmodifiableList(results); return new ResultSet() { diff --git a/java/com/google/gerrit/elasticsearch/BUILD b/java/com/google/gerrit/elasticsearch/BUILD index f5ada859ea..b582a39fe8 100644 --- a/java/com/google/gerrit/elasticsearch/BUILD +++ b/java/com/google/gerrit/elasticsearch/BUILD @@ -16,15 +16,16 @@ java_library( "//lib:protobuf", "//lib/commons:codec", "//lib/commons:lang", - "//lib/elasticsearch", "//lib/elasticsearch:joda-time", + "//lib/elasticsearch-rest-client", "//lib/guice", "//lib/guice:guice-assistedinject", - "//lib/jest", - "//lib/jest:jest-common", + "//lib/httpcomponents:httpasyncclient", + "//lib/httpcomponents:httpclient", + "//lib/httpcomponents:httpcore", + "//lib/httpcomponents:httpcore-nio", + "//lib/jackson:jackson-core", "//lib/jgit/org.eclipse.jgit:jgit", "//lib/log:api", - "//lib/lucene:lucene-analyzers-common", - "//lib/lucene:lucene-core", ], ) diff --git a/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java index 8ac0109d30..9fc1eb4426 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java @@ -31,19 +31,18 @@ import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.account.AccountField; import com.google.gerrit.server.index.account.AccountIndex; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; -import io.searchbox.client.JestResult; -import io.searchbox.core.Bulk; -import io.searchbox.core.Bulk.Builder; -import io.searchbox.core.search.sort.Sort; -import io.searchbox.core.search.sort.Sort.Sorting; import java.io.IOException; import java.util.Set; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; import org.eclipse.jgit.lib.Config; +import org.elasticsearch.client.Response; public class ElasticAccountIndex extends AbstractElasticIndex implements AccountIndex { @@ -65,7 +64,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex accountCache, - JestClientBuilder clientBuilder, + ElasticRestClientBuilder clientBuilder, @Assisted Schema schema) { super(cfg, sitePaths, schema, clientBuilder, ACCOUNTS); this.accountCache = accountCache; @@ -74,33 +73,31 @@ public class ElasticAccountIndex extends AbstractElasticIndex getSource(Predicate p, QueryOptions opts) throws QueryParseException { - Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC); - sort.setIgnoreUnmapped(); - return new ElasticQuerySource(p, opts.filterFields(IndexUtils::accountFields), ACCOUNTS, sort); + JsonArray sortArray = getSortArray(AccountField.ID.getName()); + return new ElasticQuerySource( + p, opts.filterFields(IndexUtils::accountFields), ACCOUNTS, sortArray); } @Override - protected Builder addActions(Builder builder, Account.Id c) { - return builder.addAction(delete(ACCOUNTS, c)); + protected String addActions(Account.Id c) { + return delete(ACCOUNTS, c); } @Override diff --git a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java index 58a298eac0..c0ff2f6b69 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java @@ -24,7 +24,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.codec.binary.Base64.decodeBase64; import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -61,18 +60,16 @@ import com.google.gwtorm.server.OrmException; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; -import io.searchbox.client.JestResult; -import io.searchbox.core.Bulk; -import io.searchbox.core.Bulk.Builder; -import io.searchbox.core.search.sort.Sort; -import io.searchbox.core.search.sort.Sort.Sorting; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; import org.eclipse.jgit.lib.Config; +import org.elasticsearch.client.Response; /** Secondary index implementation using Elasticsearch. */ class ElasticChangeIndex extends AbstractElasticIndex @@ -102,7 +99,7 @@ class ElasticChangeIndex extends AbstractElasticIndex Provider db, ChangeData.Factory changeDataFactory, SitePaths sitePaths, - JestClientBuilder clientBuilder, + ElasticRestClientBuilder clientBuilder, @Assisted Schema schema) { super(cfg, sitePaths, schema, clientBuilder, CHANGES); this.db = db; @@ -127,20 +124,17 @@ class ElasticChangeIndex extends AbstractElasticIndex throw new IOException(e); } - Bulk bulk = - new Bulk.Builder() - .defaultIndex(indexName) - .defaultType("changes") - .addAction(insert(insertIndex, cd)) - .addAction(delete(deleteIndex, cd.getId())) - .refresh(true) - .build(); - JestResult result = client.execute(bulk); - if (!result.isSucceeded()) { + String bulk = toAction(insertIndex, getId(cd), INDEX); + bulk += toDoc(cd); + bulk += toAction(deleteIndex, cd.getId().toString(), DELETE); + + String uri = getURI(CHANGES, BULK); + Response response = performRequest(HttpPost.METHOD_NAME, bulk, uri, getRefreshParam()); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { throw new IOException( String.format( - "Failed to replace change %s in index %s: %s", - cd.getId(), indexName, result.getErrorMessage())); + "Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode)); } } @@ -156,20 +150,28 @@ class ElasticChangeIndex extends AbstractElasticIndex indexes.add(CLOSED_CHANGES); } - List sorts = - ImmutableList.of( - new Sort(ChangeField.UPDATED.getName(), Sorting.DESC), - new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC)); - for (Sort sort : sorts) { - sort.setIgnoreUnmapped(); - } QueryOptions filteredOpts = opts.filterFields(IndexUtils::changeFields); - return new ElasticQuerySource(p, filteredOpts, indexes, sorts); + return new ElasticQuerySource(p, filteredOpts, getURI(indexes), getSortArray()); + } + + private JsonArray getSortArray() { + JsonObject properties = new JsonObject(); + properties.addProperty(ORDER, "desc"); + properties.addProperty(IGNORE_UNMAPPED, true); + + JsonArray sortArray = new JsonArray(); + addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray); + addNamedElement(ChangeField.LEGACY_ID.getName(), properties, sortArray); + return sortArray; + } + + private String getURI(List types) { + return String.join(",", types); } @Override - protected Builder addActions(Builder builder, Id c) { - return builder.addAction(delete(OPEN_CHANGES, c)).addAction(delete(OPEN_CHANGES, c)); + protected String addActions(Id c) { + return delete(OPEN_CHANGES, c) + delete(CLOSED_CHANGES, c); } @Override diff --git a/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java index 7ae49c7717..4cca280362 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java +++ b/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java @@ -18,13 +18,12 @@ import com.google.common.base.MoreObjects; import com.google.gerrit.server.config.GerritServerConfig; import com.google.inject.Inject; import com.google.inject.Singleton; -import java.net.MalformedURLException; -import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.http.HttpHost; import org.eclipse.jgit.lib.Config; @Singleton @@ -33,7 +32,7 @@ class ElasticConfiguration { private static final String DEFAULT_PORT = "9200"; private static final String DEFAULT_PROTOCOL = "http"; - final List urls; + final List urls; final String username; final String password; final boolean requestCompression; @@ -59,14 +58,18 @@ class ElasticConfiguration { Set subsections = cfg.getSubsections("elasticsearch"); if (subsections.isEmpty()) { - this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT)); + HttpHost httpHost = + new HttpHost(DEFAULT_HOST, Integer.valueOf(DEFAULT_PORT), DEFAULT_PROTOCOL); + this.urls = Collections.singletonList(httpHost); } else { this.urls = new ArrayList<>(subsections.size()); for (String subsection : subsections) { String port = getString(cfg, subsection, "port", DEFAULT_PORT); String host = getString(cfg, subsection, "hostname", DEFAULT_HOST); String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL); - this.urls.add(buildUrl(protocol, host, port)); + + HttpHost httpHost = new HttpHost(host, Integer.valueOf(port), protocol); + this.urls.add(httpHost); } } } @@ -74,19 +77,4 @@ class ElasticConfiguration { private String getString(Config cfg, String subsection, String name, String defaultValue) { return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue); } - - private String buildUrl(String protocol, String hostname, String port) { - try { - return new URL(protocol, hostname, Integer.parseInt(port), "").toString(); - } catch (MalformedURLException | NumberFormatException e) { - throw new RuntimeException( - "Cannot build url to Elasticsearch from values: protocol=" - + protocol - + " hostname=" - + hostname - + " port=" - + port, - e); - } - } } diff --git a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java index fa46b3ec11..48d8d39abd 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java @@ -29,18 +29,18 @@ import com.google.gerrit.server.group.InternalGroup; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.group.GroupField; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; -import io.searchbox.client.JestResult; -import io.searchbox.core.Bulk; -import io.searchbox.core.Bulk.Builder; -import io.searchbox.core.search.sort.Sort; import java.io.IOException; import java.util.Set; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; import org.eclipse.jgit.lib.Config; +import org.elasticsearch.client.Response; public class ElasticGroupIndex extends AbstractElasticIndex implements GroupIndex { @@ -62,7 +62,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex groupCache, - JestClientBuilder clientBuilder, + ElasticRestClientBuilder clientBuilder, @Assisted Schema schema) { super(cfg, sitePaths, schema, clientBuilder, GROUPS); this.groupCache = groupCache; @@ -71,33 +71,30 @@ public class ElasticGroupIndex extends AbstractElasticIndex getSource(Predicate p, QueryOptions opts) throws QueryParseException { - Sort sort = new Sort(GroupField.UUID.getName(), Sort.Sorting.ASC); - sort.setIgnoreUnmapped(); - return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), GROUPS, sort); + JsonArray sortArray = getSortArray(GroupField.UUID.getName()); + return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), GROUPS, sortArray); } @Override - protected Builder addActions(Builder builder, AccountGroup.UUID c) { - return builder.addAction(delete(GROUPS, c)); + protected String addActions(AccountGroup.UUID c) { + return delete(GROUPS, c); } @Override diff --git a/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java b/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java index b73b37f556..61d59adbd2 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java +++ b/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java @@ -16,31 +16,37 @@ package com.google.gerrit.elasticsearch; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.inject.Inject; import com.google.inject.Singleton; -import io.searchbox.client.JestResult; -import io.searchbox.client.http.JestHttpClient; -import io.searchbox.indices.aliases.GetAliases; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map.Entry; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; @Singleton class ElasticIndexVersionDiscovery { - private final JestHttpClient client; + private final RestClient client; @Inject - ElasticIndexVersionDiscovery(JestClientBuilder clientBuilder) { + ElasticIndexVersionDiscovery(ElasticRestClientBuilder clientBuilder) { this.client = clientBuilder.build(); } List discover(String prefix, String indexName) throws IOException { String name = prefix + indexName + "_"; - JestResult result = client.execute(new GetAliases.Builder().addIndex(name + "*").build()); - if (result.isSucceeded()) { - JsonObject object = result.getJsonObject().getAsJsonObject(); + Response response = client.performRequest(HttpGet.METHOD_NAME, name + "*/_aliases"); + + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_OK) { + String content = AbstractElasticIndex.getContent(response); + JsonObject object = new JsonParser().parse(content).getAsJsonObject(); + List versions = new ArrayList<>(object.size()); for (Entry entry : object.entrySet()) { versions.add(entry.getKey().replace(name, "")); diff --git a/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java index e9194c769e..5ac337b472 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java @@ -29,19 +29,18 @@ import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.project.ProjectCache; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; -import io.searchbox.client.JestResult; -import io.searchbox.core.Bulk; -import io.searchbox.core.Bulk.Builder; -import io.searchbox.core.search.sort.Sort; -import io.searchbox.core.search.sort.Sort.Sorting; import java.io.IOException; import java.util.Set; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; import org.eclipse.jgit.lib.Config; +import org.elasticsearch.client.Response; public class ElasticProjectIndex extends AbstractElasticIndex implements ProjectIndex { @@ -63,7 +62,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex projectCache, - JestClientBuilder clientBuilder, + ElasticRestClientBuilder clientBuilder, @Assisted Schema schema) { super(cfg, sitePaths, schema, clientBuilder, PROJECTS); this.projectCache = projectCache; @@ -72,33 +71,31 @@ public class ElasticProjectIndex extends AbstractElasticIndex getSource(Predicate p, QueryOptions opts) throws QueryParseException { - Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC); - sort.setIgnoreUnmapped(); - return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), PROJECTS, sort); + JsonArray sortArray = getSortArray(ProjectField.NAME.getName()); + return new ElasticQuerySource( + p, opts.filterFields(IndexUtils::projectFields), PROJECTS, sortArray); } @Override - protected Builder addActions(Builder builder, Project.NameKey nameKey) { - return builder.addAction(delete(PROJECTS, nameKey)); + protected String addActions(Project.NameKey nameKey) { + return delete(PROJECTS, nameKey); } @Override diff --git a/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java index 6905cf4b13..2a97e2ef39 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java +++ b/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java @@ -14,6 +14,9 @@ package com.google.gerrit.elasticsearch; +import com.google.gerrit.elasticsearch.builders.BoolQueryBuilder; +import com.google.gerrit.elasticsearch.builders.QueryBuilder; +import com.google.gerrit.elasticsearch.builders.QueryBuilders; import com.google.gerrit.index.FieldDef; import com.google.gerrit.index.FieldType; import com.google.gerrit.index.query.AndPredicate; @@ -28,10 +31,6 @@ import com.google.gerrit.index.query.RegexPredicate; import com.google.gerrit.index.query.TimestampRangePredicate; import com.google.gerrit.server.query.change.AfterPredicate; import java.time.Instant; -import org.apache.lucene.search.BooleanQuery; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; public class ElasticQueryBuilder { @@ -52,27 +51,19 @@ public class ElasticQueryBuilder { } private BoolQueryBuilder and(Predicate p) throws QueryParseException { - try { - BoolQueryBuilder b = QueryBuilders.boolQuery(); - for (Predicate c : p.getChildren()) { - b.must(toQueryBuilder(c)); - } - return b; - } catch (BooleanQuery.TooManyClauses e) { - throw new QueryParseException("cannot create query for index: " + p, e); + BoolQueryBuilder b = QueryBuilders.boolQuery(); + for (Predicate c : p.getChildren()) { + b.must(toQueryBuilder(c)); } + return b; } private BoolQueryBuilder or(Predicate p) throws QueryParseException { - try { - BoolQueryBuilder q = QueryBuilders.boolQuery(); - for (Predicate c : p.getChildren()) { - q.should(toQueryBuilder(c)); - } - return q; - } catch (BooleanQuery.TooManyClauses e) { - throw new QueryParseException("cannot create query for index: " + p, e); + BoolQueryBuilder q = QueryBuilders.boolQuery(); + for (Predicate c : p.getChildren()) { + q.should(toQueryBuilder(c)); } + return q; } private QueryBuilder not(Predicate p) throws QueryParseException { diff --git a/java/com/google/gerrit/elasticsearch/ElasticRestClientBuilder.java b/java/com/google/gerrit/elasticsearch/ElasticRestClientBuilder.java new file mode 100644 index 0000000000..faf1c71b55 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/ElasticRestClientBuilder.java @@ -0,0 +1,58 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +@Singleton +class ElasticRestClientBuilder { + + private final HttpHost[] hosts; + private final String username; + private final String password; + + @Inject + ElasticRestClientBuilder(ElasticConfiguration cfg) { + hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]); + username = cfg.username; + password = cfg.password; + } + + RestClient build() { + RestClientBuilder builder = RestClient.builder(hosts); + setConfiguredCredentialsIfAny(builder); + return builder.build(); + } + + private void setConfiguredCredentialsIfAny(RestClientBuilder builder) { + if (username != null && password != null) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + builder.setHttpClientConfigCallback( + (HttpAsyncClientBuilder httpClientBuilder) -> + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + } +} diff --git a/java/com/google/gerrit/elasticsearch/JestClientBuilder.java b/java/com/google/gerrit/elasticsearch/JestClientBuilder.java deleted file mode 100644 index c548cb9b1e..0000000000 --- a/java/com/google/gerrit/elasticsearch/JestClientBuilder.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.elasticsearch; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import io.searchbox.client.JestClientFactory; -import io.searchbox.client.config.HttpClientConfig; -import io.searchbox.client.config.HttpClientConfig.Builder; -import io.searchbox.client.http.JestHttpClient; -import java.util.concurrent.TimeUnit; - -@Singleton -class JestClientBuilder { - private final ElasticConfiguration cfg; - - @Inject - JestClientBuilder(ElasticConfiguration cfg) { - this.cfg = cfg; - } - - JestHttpClient build() { - JestClientFactory factory = new JestClientFactory(); - Builder builder = - new HttpClientConfig.Builder(cfg.urls) - .multiThreaded(true) - .discoveryEnabled(false) - .connTimeout((int) cfg.connectionTimeout) - .maxConnectionIdleTime(cfg.maxConnectionIdleTime, cfg.maxConnectionIdleUnit) - .maxTotalConnection(cfg.maxTotalConnection) - .readTimeout(cfg.readTimeout) - .requestCompressionEnabled(cfg.requestCompression) - .discoveryFrequency(1L, TimeUnit.MINUTES); - - if (cfg.username != null && cfg.password != null) { - builder.defaultCredentials(cfg.username, cfg.password); - } - - factory.setHttpClientConfig(builder.build()); - return (JestHttpClient) factory.getObject(); - } -} diff --git a/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java new file mode 100644 index 0000000000..85072b37d6 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java @@ -0,0 +1,88 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A Query that matches documents matching boolean combinations of other queries. A trimmed down + * version of {@link org.elasticsearch.index.query.BoolQueryBuilder} for this very package. + */ +public class BoolQueryBuilder extends QueryBuilder { + + private final List mustClauses = new ArrayList<>(); + + private final List mustNotClauses = new ArrayList<>(); + + private final List filterClauses = new ArrayList<>(); + + private final List shouldClauses = new ArrayList<>(); + + /** + * Adds a query that must appear in the matching documents and will contribute to scoring. + */ + public BoolQueryBuilder must(QueryBuilder queryBuilder) { + mustClauses.add(queryBuilder); + return this; + } + + /** + * Adds a query that must not appear in the matching documents and will not contribute to + * scoring. + */ + public BoolQueryBuilder mustNot(QueryBuilder queryBuilder) { + mustNotClauses.add(queryBuilder); + return this; + } + + /** + * Adds a query that should appear in the matching documents. For a boolean query with no + * MUST clauses one or more SHOULD clauses must match a document for the + * BooleanQuery to match. + */ + public BoolQueryBuilder should(QueryBuilder queryBuilder) { + shouldClauses.add(queryBuilder); + return this; + } + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("bool"); + doXArrayContent("must", mustClauses, builder); + doXArrayContent("filter", filterClauses, builder); + doXArrayContent("must_not", mustNotClauses, builder); + doXArrayContent("should", shouldClauses, builder); + builder.endObject(); + } + + private void doXArrayContent(String field, List clauses, XContentBuilder builder) + throws IOException { + if (clauses.isEmpty()) { + return; + } + if (clauses.size() == 1) { + builder.field(field); + clauses.get(0).toXContent(builder); + } else { + builder.startArray(field); + for (QueryBuilder clause : clauses) { + clause.toXContent(builder); + } + builder.endArray(); + } + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java new file mode 100644 index 0000000000..6fa0af48aa --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java @@ -0,0 +1,37 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * Constructs a query that only match on documents that the field has a value in them. A trimmed + * down version of {@link org.elasticsearch.index.query.ExistsQueryBuilder} for this very package. + */ +class ExistsQueryBuilder extends QueryBuilder { + + private final String name; + + ExistsQueryBuilder(String name) { + this.name = name; + } + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("exists"); + builder.field("field", name); + builder.endObject(); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java new file mode 100644 index 0000000000..b07af34a2b --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java @@ -0,0 +1,30 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * A query that matches on all documents. A trimmed down version of {@link + * org.elasticsearch.index.query.MatchAllQueryBuilder} for this very package. + */ +class MatchAllQueryBuilder extends QueryBuilder { + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("match_all"); + builder.endObject(); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java new file mode 100644 index 0000000000..1b1c51a474 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java @@ -0,0 +1,64 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; +import java.util.Locale; + +/** + * Match query is a query that analyzes the text and constructs a query as the result of the + * analysis. It can construct different queries based on the type provided. A trimmed down version + * of {@link org.elasticsearch.index.query.MatchQueryBuilder} for this very package. + */ +class MatchQueryBuilder extends QueryBuilder { + + enum Type { + /** The text is analyzed and used as a phrase query. */ + PHRASE, + /** The text is analyzed and used in a phrase query, with the last term acting as a prefix. */ + PHRASE_PREFIX + } + + private final String name; + + private final Object text; + + private Type type; + + /** Constructs a new text query. */ + MatchQueryBuilder(String name, Object text) { + this.name = name; + this.text = text; + } + + /** Sets the type of the text query. */ + MatchQueryBuilder type(Type type) { + this.type = type; + return this; + } + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("match"); + builder.startObject(name); + + builder.field("query", text); + if (type != null) { + builder.field("type", type.toString().toLowerCase(Locale.ENGLISH)); + } + builder.endObject(); + builder.endObject(); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java new file mode 100644 index 0000000000..cb6476eca1 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java @@ -0,0 +1,34 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * A trimmed down version of {@link org.elasticsearch.index.query.QueryBuilder} for this very + * package. + */ +public abstract class QueryBuilder { + + protected QueryBuilder() {} + + protected void toXContent(XContentBuilder builder) throws IOException { + builder.startObject(); + doXContent(builder); + builder.endObject(); + } + + protected abstract void doXContent(XContentBuilder builder) throws IOException; +} diff --git a/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java b/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java new file mode 100644 index 0000000000..c97d70ea4b --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java @@ -0,0 +1,102 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +/** + * A static factory for simple "import static" usage. A trimmed down version of {@link + * org.elasticsearch.index.query.QueryBuilders} for this very package. + */ +public abstract class QueryBuilders { + + /** A query that match on all documents. */ + public static MatchAllQueryBuilder matchAllQuery() { + return new MatchAllQueryBuilder(); + } + + /** + * Creates a text query with type "PHRASE" for the provided field name and text. + * + * @param name The field name. + * @param text The query text (to be analyzed). + */ + public static MatchQueryBuilder matchPhraseQuery(String name, Object text) { + return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE); + } + + /** + * Creates a match query with type "PHRASE_PREFIX" for the provided field name and text. + * + * @param name The field name. + * @param text The query text (to be analyzed). + */ + public static MatchQueryBuilder matchPhrasePrefixQuery(String name, Object text) { + return new MatchQueryBuilder(name, text).type(MatchQueryBuilder.Type.PHRASE_PREFIX); + } + + /** + * A Query that matches documents containing a term. + * + * @param name The name of the field + * @param value The value of the term + */ + public static TermQueryBuilder termQuery(String name, String value) { + return new TermQueryBuilder(name, value); + } + + /** + * A Query that matches documents containing a term. + * + * @param name The name of the field + * @param value The value of the term + */ + public static TermQueryBuilder termQuery(String name, int value) { + return new TermQueryBuilder(name, value); + } + + /** + * A Query that matches documents within an range of terms. + * + * @param name The field name + */ + public static RangeQueryBuilder rangeQuery(String name) { + return new RangeQueryBuilder(name); + } + + /** + * A Query that matches documents containing terms with a specified regular expression. + * + * @param name The name of the field + * @param regexp The regular expression + */ + public static RegexpQueryBuilder regexpQuery(String name, String regexp) { + return new RegexpQueryBuilder(name, regexp); + } + + /** A Query that matches documents matching boolean combinations of other queries. */ + public static BoolQueryBuilder boolQuery() { + return new BoolQueryBuilder(); + } + + /** + * A filter to filter only documents where a field exists in them. + * + * @param name The name of the field + */ + public static ExistsQueryBuilder existsQuery(String name) { + return new ExistsQueryBuilder(name); + } + + private QueryBuilders() {} +} diff --git a/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java b/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java new file mode 100644 index 0000000000..f57ef3dc6c --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java @@ -0,0 +1,35 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * A trimmed down and further altered version of {@link + * org.elasticsearch.action.support.QuerySourceBuilder} for this very package. + */ +class QuerySourceBuilder { + + private final QueryBuilder queryBuilder; + + QuerySourceBuilder(QueryBuilder queryBuilder) { + this.queryBuilder = queryBuilder; + } + + void innerToXContent(XContentBuilder builder) throws IOException { + builder.field("query"); + queryBuilder.toXContent(builder); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java new file mode 100644 index 0000000000..6049b7298d --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java @@ -0,0 +1,88 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * A Query that matches documents within an range of terms. A trimmed down version of {@link + * org.elasticsearch.index.query.RangeQueryBuilder} for this very package. + */ +public class RangeQueryBuilder extends QueryBuilder { + + private final String name; + private Object from; + private Object to; + private boolean includeLower = true; + private boolean includeUpper = true; + + /** + * A Query that matches documents within an range of terms. + * + * @param name The field name + */ + RangeQueryBuilder(String name) { + this.name = name; + } + + /** The from part of the range query. Null indicates unbounded. */ + public RangeQueryBuilder gt(Object from) { + this.from = from; + this.includeLower = false; + return this; + } + + /** The from part of the range query. Null indicates unbounded. */ + public RangeQueryBuilder gte(Object from) { + this.from = from; + this.includeLower = true; + return this; + } + + /** The from part of the range query. Null indicates unbounded. */ + public RangeQueryBuilder gte(int from) { + this.from = from; + this.includeLower = true; + return this; + } + + /** The to part of the range query. Null indicates unbounded. */ + public RangeQueryBuilder lte(Object to) { + this.to = to; + this.includeUpper = true; + return this; + } + + /** The to part of the range query. Null indicates unbounded. */ + public RangeQueryBuilder lte(int to) { + this.to = to; + this.includeUpper = true; + return this; + } + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("range"); + builder.startObject(name); + + builder.field("from", from); + builder.field("to", to); + builder.field("include_lower", includeLower); + builder.field("include_upper", includeUpper); + + builder.endObject(); + builder.endObject(); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java new file mode 100644 index 0000000000..f4d1a7e1e2 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java @@ -0,0 +1,50 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * A Query that does fuzzy matching for a specific value. A trimmed down version of {@link + * org.elasticsearch.index.query.RegexpQueryBuilder} for this very package. + */ +class RegexpQueryBuilder extends QueryBuilder { + + private final String name; + private final String regexp; + + /** + * Constructs a new term query. + * + * @param name The name of the field + * @param regexp The regular expression + */ + RegexpQueryBuilder(String name, String regexp) { + this.name = name; + this.regexp = regexp; + } + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("regexp"); + builder.startObject(name); + + builder.field("value", regexp); + builder.field("flags_value", 65535); + + builder.endObject(); + builder.endObject(); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java b/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java new file mode 100644 index 0000000000..770839dbd0 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java @@ -0,0 +1,108 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; +import java.util.List; + +/** + * A search source builder allowing to easily build search source. A trimmed down and further + * altered version of {@link org.elasticsearch.search.builder.SearchSourceBuilder} for this very + * package. + */ +public class SearchSourceBuilder { + + private QuerySourceBuilder querySourceBuilder; + + private int from = -1; + + private int size = -1; + + private List fieldNames; + + /** Constructs a new search source builder. */ + public SearchSourceBuilder() {} + + /** Constructs a new search source builder with a search query. */ + public SearchSourceBuilder query(QueryBuilder query) { + if (this.querySourceBuilder == null) { + this.querySourceBuilder = new QuerySourceBuilder(query); + } + return this; + } + + /** From index to start the search from. Defaults to 0. */ + public SearchSourceBuilder from(int from) { + this.from = from; + return this; + } + + /** The number of search hits to return. Defaults to 10. */ + public SearchSourceBuilder size(int size) { + this.size = size; + return this; + } + + /** + * Sets the fields to load and return as part of the search request. If none are specified, the + * source of the document will be returned. + */ + public SearchSourceBuilder fields(List fields) { + this.fieldNames = fields; + return this; + } + + @Override + public final String toString() { + try { + XContentBuilder builder = new XContentBuilder(); + toXContent(builder); + return builder.string(); + } catch (IOException ioe) { + return ""; + } + } + + private void toXContent(XContentBuilder builder) throws IOException { + builder.startObject(); + innerToXContent(builder); + builder.endObject(); + } + + private void innerToXContent(XContentBuilder builder) throws IOException { + if (from != -1) { + builder.field("from", from); + } + if (size != -1) { + builder.field("size", size); + } + + if (querySourceBuilder != null) { + querySourceBuilder.innerToXContent(builder); + } + + if (fieldNames != null) { + if (fieldNames.size() == 1) { + builder.field("fields", fieldNames.get(0)); + } else { + builder.startArray("fields"); + for (String fieldName : fieldNames) { + builder.value(fieldName); + } + builder.endArray(); + } + } + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java b/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java new file mode 100644 index 0000000000..ed02af3333 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java @@ -0,0 +1,66 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import java.io.IOException; + +/** + * A Query that matches documents containing a term. A trimmed down version of {@link + * org.elasticsearch.index.query.TermQueryBuilder} for this very package. + */ +class TermQueryBuilder extends QueryBuilder { + + private final String name; + + private final Object value; + + /** + * Constructs a new term query. + * + * @param name The name of the field + * @param value The value of the term + */ + TermQueryBuilder(String name, String value) { + this(name, (Object) value); + } + + /** + * Constructs a new term query. + * + * @param name The name of the field + * @param value The value of the term + */ + TermQueryBuilder(String name, int value) { + this(name, (Object) value); + } + + /** + * Constructs a new term query. + * + * @param name The name of the field + * @param value The value of the term + */ + private TermQueryBuilder(String name, Object value) { + this.name = name; + this.value = value; + } + + @Override + protected void doXContent(XContentBuilder builder) throws IOException { + builder.startObject("term"); + builder.field(name, value); + builder.endObject(); + } +} diff --git a/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java b/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java new file mode 100644 index 0000000000..4fad323249 --- /dev/null +++ b/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java @@ -0,0 +1,170 @@ +// Copyright (C) 2018 The Android Open Source Project, 2009-2015 Elasticsearch +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch.builders; + +import static java.time.format.DateTimeFormatter.ISO_INSTANT; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.google.common.base.Charsets; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.util.Date; + +/** + * A trimmed down and further altered version of {@link + * org.elasticsearch.common.xcontent.XContentBuilder} for this very package. + */ +public final class XContentBuilder implements Closeable { + + private final JsonGenerator generator; + + private final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + + /** + * Constructs a new builder. Make sure to call {@link #close()} when the builder is done with. + * Inspired from {@link org.elasticsearch.common.xcontent.json.JsonXContent} static block. + */ + public XContentBuilder() throws IOException { + JsonFactory jsonFactory = new JsonFactory(); + jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true); + jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + jsonFactory.configure( + JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, + false); // this trips on many mappings now... + this.generator = jsonFactory.createGenerator(bos, JsonEncoding.UTF8); + } + + public XContentBuilder startObject(String name) throws IOException { + field(name); + startObject(); + return this; + } + + public XContentBuilder startObject() throws IOException { + generator.writeStartObject(); + return this; + } + + public XContentBuilder endObject() throws IOException { + generator.writeEndObject(); + return this; + } + + public void startArray(String name) throws IOException { + field(name); + startArray(); + } + + private void startArray() throws IOException { + generator.writeStartArray(); + } + + public void endArray() throws IOException { + generator.writeEndArray(); + } + + public XContentBuilder field(String name) throws IOException { + generator.writeFieldName(name); + return this; + } + + public XContentBuilder field(String name, String value) throws IOException { + field(name); + generator.writeString(value); + return this; + } + + public XContentBuilder field(String name, int value) throws IOException { + field(name); + generator.writeNumber(value); + return this; + } + + public XContentBuilder field(String name, Iterable value) throws IOException { + startArray(name); + for (Object o : value) { + value(o); + } + endArray(); + return this; + } + + public XContentBuilder field(String name, Object value) throws IOException { + field(name); + writeValue(value); + return this; + } + + public XContentBuilder value(Object value) throws IOException { + writeValue(value); + return this; + } + + public XContentBuilder field(String name, boolean value) throws IOException { + field(name); + generator.writeBoolean(value); + return this; + } + + public XContentBuilder value(String value) throws IOException { + generator.writeString(value); + return this; + } + + @Override + public void close() { + try { + generator.close(); + } catch (IOException e) { + // ignore + } + } + + /** Returns a string representation of the builder (only applicable for text based xcontent). */ + public String string() { + close(); + byte[] bytesArray = bos.toByteArray(); + return new String(bytesArray, Charsets.UTF_8); + } + + private void writeValue(Object value) throws IOException { + if (value == null) { + generator.writeNull(); + return; + } + Class type = value.getClass(); + if (type == String.class) { + generator.writeString((String) value); + } else if (type == Integer.class) { + generator.writeNumber(((Integer) value)); + } else if (type == byte[].class) { + generator.writeBinary((byte[]) value); + } else if (value instanceof Date) { + generator.writeString(ISO_INSTANT.format(((Date) value).toInstant())); + } else { + // if this is a "value" object, like enum, DistanceUnit, ..., just toString it + // yea, it can be misleading when toString a Java class, but really, jackson should be used in + // that case + generator.writeString(value.toString()); + // throw new ElasticsearchIllegalArgumentException("type not supported for generic value + // conversion: " + type); + } + } +} diff --git a/lib/LICENSE-elasticsearch b/lib/LICENSE-elasticsearch new file mode 100644 index 0000000000..23cae9e206 --- /dev/null +++ b/lib/LICENSE-elasticsearch @@ -0,0 +1,5 @@ +Elasticsearch +Copyright 2009-2015 Elasticsearch + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). diff --git a/lib/elasticsearch-rest-client/BUILD b/lib/elasticsearch-rest-client/BUILD new file mode 100644 index 0000000000..c6357d083e --- /dev/null +++ b/lib/elasticsearch-rest-client/BUILD @@ -0,0 +1,8 @@ +package(default_visibility = ["//visibility:public"]) + +java_library( + name = "elasticsearch-rest-client", + data = ["//lib:LICENSE-elasticsearch"], + visibility = ["//visibility:public"], + exports = ["@elasticsearch-rest-client//jar"], +) diff --git a/lib/elasticsearch/BUILD b/lib/elasticsearch/BUILD index 13c033ecdf..b564c55176 100644 --- a/lib/elasticsearch/BUILD +++ b/lib/elasticsearch/BUILD @@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"]) java_library( name = "elasticsearch", + testonly = 1, data = ["//lib:LICENSE-Apache2.0"], exports = ["@elasticsearch//jar"], runtime_deps = [ diff --git a/lib/jest/BUILD b/lib/jest/BUILD deleted file mode 100644 index 169f27157d..0000000000 --- a/lib/jest/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -java_library( - name = "jest-common", - data = ["//lib:LICENSE-Apache2.0"], - visibility = ["//visibility:public"], - exports = ["@jest_common//jar"], - runtime_deps = [ - "//lib/commons:lang3", - ], -) - -java_library( - name = "jest", - data = ["//lib:LICENSE-Apache2.0"], - visibility = ["//visibility:public"], - exports = ["@jest//jar"], - runtime_deps = [ - "//lib/httpcomponents:httpasyncclient", - "//lib/httpcomponents:httpclient", - "//lib/httpcomponents:httpcore-nio", - ], -) diff --git a/tools/eclipse/BUILD b/tools/eclipse/BUILD index 546c75e7e9..62e6bf61ca 100644 --- a/tools/eclipse/BUILD +++ b/tools/eclipse/BUILD @@ -10,6 +10,7 @@ load( TEST_DEPS = [ "//gerrit-gwtui:ui_tests", "//javatests/com/google/gerrit/server:server_tests", + "//lib/elasticsearch", ] DEPS = [