Add support for secondary index with Elasticsearch
Add support for secondary index on Elasticsearch via the REST API using the Jest client [1]. Because Elasticsearch uses different version of Lucene we add another maven_jar's in gerrit-elasticsearch BUCK. Fortunately both versions have compatible API, this way we are able to compile and run Gerrit. All tests for changes index passes, but they need to use Lucene's based account index. [1] http://www.searchly.com/documentation/developer-api-guide/java-jest/ TODO: Add support for online reindex TODO: Add support for schema upgrades Also-By: Janice Agustin <janice.agustin@ericsson.com> Also-By: Olga Grinberg <olga.grinberg@ericsson.com> Also-By: Dariusz Luksza <dluksza@collab.net> Change-Id: I5e4fc08ce34d33c090c9e0bf320de1b17309f774
This commit is contained in:
David Pursehouse
committed by
David Pursehouse
David Pursehouse
parent
b4d30002d6
commit
8e72f5301b
@@ -0,0 +1,207 @@
|
||||
// Copyright (C) 2014 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.elasticsearch;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.gerrit.lucene.AbstractLuceneIndex.setReady;
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.gerrit.server.config.GerritServerConfig;
|
||||
import com.google.gerrit.server.config.SitePaths;
|
||||
import com.google.gerrit.server.index.FieldDef.FillArgs;
|
||||
import com.google.gerrit.server.index.Index;
|
||||
import com.google.gerrit.server.index.Schema;
|
||||
import com.google.gerrit.server.index.Schema.Values;
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.assistedinject.Assisted;
|
||||
|
||||
import org.eclipse.jgit.lib.Config;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import io.searchbox.client.JestClientFactory;
|
||||
import io.searchbox.client.JestResult;
|
||||
import io.searchbox.client.config.HttpClientConfig;
|
||||
import io.searchbox.client.http.JestHttpClient;
|
||||
import io.searchbox.core.Bulk;
|
||||
import io.searchbox.core.Delete;
|
||||
import io.searchbox.indices.CreateIndex;
|
||||
import io.searchbox.indices.DeleteIndex;
|
||||
import io.searchbox.indices.IndicesExists;
|
||||
|
||||
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
|
||||
private static final String DEFAULT_INDEX_NAME = "gerrit";
|
||||
|
||||
private final Schema<V> schema;
|
||||
private final FillArgs fillArgs;
|
||||
private final SitePaths sitePaths;
|
||||
|
||||
protected final boolean refresh;
|
||||
protected final String indexName;
|
||||
protected final JestHttpClient client;
|
||||
|
||||
|
||||
@Inject
|
||||
AbstractElasticIndex(@GerritServerConfig Config cfg,
|
||||
FillArgs fillArgs,
|
||||
SitePaths sitePaths,
|
||||
@Assisted Schema<V> schema) {
|
||||
this.fillArgs = fillArgs;
|
||||
this.sitePaths = sitePaths;
|
||||
this.schema = schema;
|
||||
String protocol = getRequiredConfigOption(cfg, "protocol");
|
||||
String hostname = getRequiredConfigOption(cfg, "hostname");
|
||||
String port = getRequiredConfigOption(cfg, "port");
|
||||
|
||||
this.indexName =
|
||||
firstNonNull(cfg.getString("index", null, "name"), DEFAULT_INDEX_NAME);
|
||||
|
||||
// By default Elasticsearch has a 1s delay before changes are available in
|
||||
// the index. Setting refresh(true) on calls to the index makes the index
|
||||
// refresh immediately.
|
||||
//
|
||||
// Discovery should be disabled during test mode to prevent spurious
|
||||
// connection failures caused by the client starting up and being ready
|
||||
// before the test node.
|
||||
//
|
||||
// This setting should only be set to true during testing, and is not
|
||||
// documented.
|
||||
this.refresh = cfg.getBoolean("index", "elasticsearch", "test", false);
|
||||
|
||||
String url = buildUrl(protocol, hostname, port);
|
||||
JestClientFactory factory = new JestClientFactory();
|
||||
factory.setHttpClientConfig(new HttpClientConfig
|
||||
.Builder(url)
|
||||
.multiThreaded(true)
|
||||
.discoveryEnabled(!refresh)
|
||||
.discoveryFrequency(1L, TimeUnit.MINUTES)
|
||||
.build());
|
||||
client = (JestHttpClient) factory.getObject();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Schema<V> getSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
client.shutdownClient();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markReady(boolean ready) throws IOException {
|
||||
setReady(sitePaths, indexName, schema.getVersion(), ready);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(K c) throws IOException {
|
||||
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(refresh).build();
|
||||
JestResult result = client.execute(bulk);
|
||||
if (!result.isSucceeded()) {
|
||||
throw new IOException(String.format(
|
||||
"Failed to delete change %s in index %s: %s", c, indexName,
|
||||
result.getErrorMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAll() throws IOException {
|
||||
// Delete the index, if it exists.
|
||||
JestResult result = client.execute(
|
||||
new IndicesExists.Builder(indexName).build());
|
||||
if (result.isSucceeded()) {
|
||||
result = client.execute(
|
||||
new DeleteIndex.Builder(indexName).build());
|
||||
if (!result.isSucceeded()) {
|
||||
throw new IOException(String.format(
|
||||
"Failed to delete index %s: %s", indexName,
|
||||
result.getErrorMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
// Recreate the index.
|
||||
result = client.execute(
|
||||
new CreateIndex.Builder(indexName).settings(getMappings()).build());
|
||||
if (!result.isSucceeded()) {
|
||||
String error = String.format("Failed to create index %s: %s",
|
||||
indexName, result.getErrorMessage());
|
||||
throw new IOException(error);
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract Bulk.Builder addActions(Bulk.Builder builder, K c);
|
||||
|
||||
protected abstract String getMappings();
|
||||
|
||||
protected abstract String getId(V v);
|
||||
|
||||
protected Delete delete(String type, K c) {
|
||||
String id = c.toString();
|
||||
return new Delete.Builder(id)
|
||||
.index(indexName)
|
||||
.type(type)
|
||||
.build();
|
||||
}
|
||||
|
||||
protected io.searchbox.core.Index insert(String type, V v) throws IOException {
|
||||
String id = getId(v);
|
||||
String doc = toDoc(v);
|
||||
return new io.searchbox.core.Index.Builder(doc)
|
||||
.index(indexName)
|
||||
.type(type)
|
||||
.id(id)
|
||||
.build();
|
||||
}
|
||||
|
||||
private String toDoc(V v) throws IOException {
|
||||
XContentBuilder builder = jsonBuilder().startObject();
|
||||
for (Values<V> values : schema.buildFields(v, fillArgs)) {
|
||||
String name = values.getField().getName();
|
||||
if (values.getField().isRepeatable()) {
|
||||
builder.array(name, values.getValues());
|
||||
} else {
|
||||
Object element = Iterables.getOnlyElement(values.getValues(), "");
|
||||
if (!(element instanceof String) || !((String) element).isEmpty()) {
|
||||
builder.field(name, element);
|
||||
}
|
||||
}
|
||||
}
|
||||
return builder.endObject().string();
|
||||
}
|
||||
|
||||
private String getRequiredConfigOption(Config cfg, String name) {
|
||||
String option = cfg.getString("index", null, name);
|
||||
checkState(!Strings.isNullOrEmpty(option), "index." + name + " must be supplied");
|
||||
return option;
|
||||
}
|
||||
|
||||
private String buildUrl(String protocol, String hostname, String port) {
|
||||
try {
|
||||
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
|
||||
} catch (MalformedURLException | NumberFormatException e) {
|
||||
throw new RuntimeException(
|
||||
"Cannot build url to Elasticsearch from values: protocol=" + protocol
|
||||
+ " hostname=" + hostname + " port=" + port, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
// Copyright (C) 2014 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.elasticsearch;
|
||||
|
||||
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.CLOSED_STATUSES;
|
||||
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.OPEN_STATUSES;
|
||||
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
|
||||
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
|
||||
import com.google.gerrit.lucene.LuceneChangeIndex;
|
||||
import com.google.gerrit.reviewdb.client.Account;
|
||||
import com.google.gerrit.reviewdb.client.Change;
|
||||
import com.google.gerrit.reviewdb.client.Change.Id;
|
||||
import com.google.gerrit.reviewdb.client.Project;
|
||||
import com.google.gerrit.reviewdb.server.ReviewDb;
|
||||
import com.google.gerrit.server.ReviewerSet;
|
||||
import com.google.gerrit.server.config.GerritServerConfig;
|
||||
import com.google.gerrit.server.config.SitePaths;
|
||||
import com.google.gerrit.server.index.FieldDef;
|
||||
import com.google.gerrit.server.index.FieldDef.FillArgs;
|
||||
import com.google.gerrit.server.index.FieldType;
|
||||
import com.google.gerrit.server.index.QueryOptions;
|
||||
import com.google.gerrit.server.index.Schema;
|
||||
import com.google.gerrit.server.index.change.ChangeField;
|
||||
import com.google.gerrit.server.index.change.ChangeField.ChangeProtoField;
|
||||
import com.google.gerrit.server.index.change.ChangeField.PatchSetApprovalProtoField;
|
||||
import com.google.gerrit.server.index.change.ChangeField.PatchSetProtoField;
|
||||
import com.google.gerrit.server.index.change.ChangeIndex;
|
||||
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
|
||||
import com.google.gerrit.server.query.Predicate;
|
||||
import com.google.gerrit.server.query.QueryParseException;
|
||||
import com.google.gerrit.server.query.change.ChangeData;
|
||||
import com.google.gerrit.server.query.change.ChangeDataSource;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gwtorm.protobuf.ProtobufCodec;
|
||||
import com.google.gwtorm.server.OrmException;
|
||||
import com.google.gwtorm.server.ResultSet;
|
||||
import com.google.inject.Provider;
|
||||
import com.google.inject.assistedinject.Assisted;
|
||||
import com.google.inject.assistedinject.AssistedInject;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.eclipse.jgit.lib.Config;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import io.searchbox.client.JestResult;
|
||||
import io.searchbox.core.Bulk;
|
||||
import io.searchbox.core.Bulk.Builder;
|
||||
import io.searchbox.core.Search;
|
||||
import io.searchbox.core.search.sort.Sort;
|
||||
import io.searchbox.core.search.sort.Sort.Sorting;
|
||||
|
||||
/** Secondary index implementation using Elasticsearch. */
|
||||
class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
|
||||
implements ChangeIndex {
|
||||
private static final Logger log =
|
||||
LoggerFactory.getLogger(ElasticChangeIndex.class);
|
||||
|
||||
static class ChangeMapping {
|
||||
MappingProperties openChanges;
|
||||
MappingProperties closedChanges;
|
||||
|
||||
ChangeMapping(Schema<ChangeData> schema) {
|
||||
ElasticMapping.Builder mappingBuilder = new ElasticMapping.Builder();
|
||||
for (FieldDef<?, ?> field : schema.getFields().values()) {
|
||||
String name = field.getName();
|
||||
FieldType<?> fieldType = field.getType();
|
||||
if (fieldType == FieldType.EXACT) {
|
||||
mappingBuilder.addExactField(name);
|
||||
} else if (fieldType == FieldType.TIMESTAMP) {
|
||||
mappingBuilder.addTimestamp(name);
|
||||
} else if (fieldType == FieldType.INTEGER
|
||||
|| fieldType == FieldType.INTEGER_RANGE
|
||||
|| fieldType == FieldType.LONG) {
|
||||
mappingBuilder.addNumber(name);
|
||||
} else if (fieldType == FieldType.PREFIX
|
||||
|| fieldType == FieldType.FULL_TEXT
|
||||
|| fieldType == FieldType.STORED_ONLY) {
|
||||
mappingBuilder.addString(name);
|
||||
} else {
|
||||
throw new IllegalArgumentException(
|
||||
"Unsupported filed type " + fieldType.getName());
|
||||
}
|
||||
}
|
||||
MappingProperties mapping = mappingBuilder.build();
|
||||
openChanges = mapping;
|
||||
closedChanges = mapping;
|
||||
}
|
||||
}
|
||||
|
||||
static final String OPEN_CHANGES = "open_changes";
|
||||
static final String CLOSED_CHANGES = "closed_changes";
|
||||
|
||||
private final Gson gson;
|
||||
private final ChangeMapping mapping;
|
||||
private final Provider<ReviewDb> db;
|
||||
private final ElasticQueryBuilder queryBuilder;
|
||||
private final ChangeData.Factory changeDataFactory;
|
||||
|
||||
@AssistedInject
|
||||
ElasticChangeIndex(
|
||||
@GerritServerConfig Config cfg,
|
||||
Provider<ReviewDb> db,
|
||||
ChangeData.Factory changeDataFactory,
|
||||
FillArgs fillArgs,
|
||||
SitePaths sitePaths,
|
||||
@Assisted Schema<ChangeData> schema) {
|
||||
super(cfg, fillArgs, sitePaths, schema);
|
||||
this.db = db;
|
||||
this.changeDataFactory = changeDataFactory;
|
||||
mapping = new ChangeMapping(schema);
|
||||
|
||||
this.queryBuilder = new ElasticQueryBuilder();
|
||||
this.gson = new GsonBuilder()
|
||||
.setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES).create();
|
||||
}
|
||||
|
||||
private static <T> List<T> decodeProtos(JsonObject doc, String fieldName,
|
||||
ProtobufCodec<T> codec) {
|
||||
return FluentIterable.from(doc.getAsJsonArray(fieldName))
|
||||
.transform(i -> codec.decode(Base64.decodeBase64(i.toString())))
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void replace(ChangeData cd) throws IOException {
|
||||
String deleteIndex;
|
||||
String insertIndex;
|
||||
|
||||
try {
|
||||
if (cd.change().getStatus().isOpen()) {
|
||||
insertIndex = OPEN_CHANGES;
|
||||
deleteIndex = CLOSED_CHANGES;
|
||||
} else {
|
||||
insertIndex = CLOSED_CHANGES;
|
||||
deleteIndex = OPEN_CHANGES;
|
||||
}
|
||||
} catch (OrmException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
|
||||
Bulk bulk = new Bulk.Builder()
|
||||
.defaultIndex(indexName)
|
||||
.defaultType("changes")
|
||||
.addAction(insert(insertIndex, cd))
|
||||
.addAction(delete(deleteIndex, cd.getId()))
|
||||
.refresh(refresh)
|
||||
.build();
|
||||
JestResult result = client.execute(bulk);
|
||||
if (!result.isSucceeded()) {
|
||||
throw new IOException(String.format(
|
||||
"Failed to replace change %s in index %s: %s", cd.getId(), indexName,
|
||||
result.getErrorMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChangeDataSource getSource(Predicate<ChangeData> p, QueryOptions opts)
|
||||
throws QueryParseException {
|
||||
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
|
||||
List<String> indexes = Lists.newArrayListWithCapacity(2);
|
||||
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
|
||||
indexes.add(OPEN_CHANGES);
|
||||
}
|
||||
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
|
||||
indexes.add(CLOSED_CHANGES);
|
||||
}
|
||||
return new QuerySource(indexes, p, opts);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Builder addActions(Builder builder, Id c) {
|
||||
return builder
|
||||
.addAction(delete(OPEN_CHANGES, c))
|
||||
.addAction(delete(OPEN_CHANGES, c));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getMappings() {
|
||||
return gson.toJson(ImmutableMap.of("mappings", mapping));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getId(ChangeData cd) {
|
||||
return cd.getId().toString();
|
||||
}
|
||||
|
||||
private class QuerySource implements ChangeDataSource {
|
||||
private final Search search;
|
||||
private final Set<String> fields;
|
||||
|
||||
public QuerySource(List<String> types, Predicate<ChangeData> p,
|
||||
QueryOptions opts) throws QueryParseException {
|
||||
List<Sort> sorts = ImmutableList.of(
|
||||
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
|
||||
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
|
||||
for (Sort sort : sorts) {
|
||||
sort.setIgnoreUnmapped();
|
||||
}
|
||||
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
|
||||
fields = LuceneChangeIndex.fields(opts);
|
||||
SearchSourceBuilder searchSource = new SearchSourceBuilder()
|
||||
.query(qb)
|
||||
.from(opts.start())
|
||||
.size(opts.limit())
|
||||
.fields(Lists.newArrayList(fields));
|
||||
|
||||
search = new Search.Builder(searchSource.toString())
|
||||
.addType(types)
|
||||
.addSort(sorts)
|
||||
.addIndex(indexName)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCardinality() {
|
||||
return 10;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResultSet<ChangeData> read() throws OrmException {
|
||||
try {
|
||||
List<ChangeData> results = Collections.emptyList();
|
||||
JestResult result = client.execute(search);
|
||||
if (result.isSucceeded()) {
|
||||
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
|
||||
if (obj.get("hits") != null) {
|
||||
JsonArray json = obj.getAsJsonArray("hits");
|
||||
results = Lists.newArrayListWithCapacity(json.size());
|
||||
for (int i = 0; i < json.size(); i++) {
|
||||
results.add(toChangeData(json.get(i)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error(result.getErrorMessage());
|
||||
}
|
||||
final List<ChangeData> r = Collections.unmodifiableList(results);
|
||||
return new ResultSet<ChangeData>() {
|
||||
@Override
|
||||
public Iterator<ChangeData> iterator() {
|
||||
return r.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ChangeData> toList() {
|
||||
return r;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// Do nothing.
|
||||
}
|
||||
};
|
||||
} catch (IOException e) {
|
||||
throw new OrmException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasChange() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return search.toString();
|
||||
}
|
||||
|
||||
private ChangeData toChangeData(JsonElement json) {
|
||||
JsonElement sourceElement = json.getAsJsonObject().get("_source");
|
||||
if (sourceElement == null) {
|
||||
sourceElement = json.getAsJsonObject().get("fields");
|
||||
}
|
||||
JsonObject source = sourceElement.getAsJsonObject();
|
||||
JsonElement c = source.get(ChangeField.CHANGE.getName());
|
||||
|
||||
if (c == null) {
|
||||
int id = source.get(ChangeField.LEGACY_ID.getName()).getAsInt();
|
||||
String projectName =
|
||||
source.get(ChangeField.PROJECT.getName()).getAsString();
|
||||
if (projectName == null) {
|
||||
return changeDataFactory.createOnlyWhenNoteDbDisabled(
|
||||
db.get(), new Change.Id(id));
|
||||
}
|
||||
return changeDataFactory.create(
|
||||
db.get(), new Project.NameKey(projectName), new Change.Id(id));
|
||||
}
|
||||
|
||||
ChangeData cd = changeDataFactory.create(db.get(),
|
||||
ChangeProtoField.CODEC.decode(Base64.decodeBase64(c.getAsString())));
|
||||
|
||||
// Patch sets.
|
||||
cd.setPatchSets(decodeProtos(
|
||||
source, ChangeField.PATCH_SET.getName(), PatchSetProtoField.CODEC));
|
||||
|
||||
// Approvals.
|
||||
if (source.get(ChangeField.APPROVAL.getName()) != null) {
|
||||
cd.setCurrentApprovals(decodeProtos(source,
|
||||
ChangeField.APPROVAL.getName(), PatchSetApprovalProtoField.CODEC));
|
||||
} else if (fields.contains(ChangeField.APPROVAL.getName())) {
|
||||
cd.setCurrentApprovals(Collections.emptyList());
|
||||
}
|
||||
|
||||
JsonElement addedElement = source.get(ChangeField.ADDED.getName());
|
||||
JsonElement deletedElement = source.get(ChangeField.DELETED.getName());
|
||||
if (addedElement != null && deletedElement != null) {
|
||||
// Changed lines.
|
||||
int added = addedElement.getAsInt();
|
||||
int deleted = deletedElement.getAsInt();
|
||||
if (added != 0 && deleted != 0) {
|
||||
cd.setChangedLines(added, deleted);
|
||||
}
|
||||
}
|
||||
|
||||
// Mergeable.
|
||||
JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName());
|
||||
if (mergeableElement != null) {
|
||||
String mergeable = mergeableElement.getAsString();
|
||||
if ("1".equals(mergeable)) {
|
||||
cd.setMergeable(true);
|
||||
} else if ("0".equals(mergeable)) {
|
||||
cd.setMergeable(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Reviewed-by.
|
||||
if (source.get(ChangeField.REVIEWEDBY.getName()) != null) {
|
||||
JsonArray reviewedBy =
|
||||
source.get(ChangeField.REVIEWEDBY.getName()).getAsJsonArray();
|
||||
if (reviewedBy.size() > 0) {
|
||||
Set<Account.Id> accounts =
|
||||
Sets.newHashSetWithExpectedSize(reviewedBy.size());
|
||||
for (int i = 0; i < reviewedBy.size() ; i++) {
|
||||
int aId = reviewedBy.get(i).getAsInt();
|
||||
if (reviewedBy.size() == 1 && aId == ChangeField.NOT_REVIEWED) {
|
||||
break;
|
||||
}
|
||||
accounts.add(new Account.Id(aId));
|
||||
}
|
||||
cd.setReviewedBy(accounts);
|
||||
}
|
||||
} else if (fields.contains(ChangeField.REVIEWEDBY.getName())) {
|
||||
cd.setReviewedBy(Collections.emptySet());
|
||||
}
|
||||
|
||||
if (source.get(ChangeField.REVIEWER.getName()) != null) {
|
||||
cd.setReviewers(
|
||||
ChangeField.parseReviewerFieldValues(FluentIterable
|
||||
.from(
|
||||
source.get(ChangeField.REVIEWER.getName()).getAsJsonArray())
|
||||
.transform(JsonElement::getAsString)));
|
||||
} else if (fields.contains(ChangeField.REVIEWER.getName())) {
|
||||
cd.setReviewers(ReviewerSet.empty());
|
||||
}
|
||||
|
||||
return cd;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// Copyright (C) 2014 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.elasticsearch;
|
||||
|
||||
import com.google.gerrit.lifecycle.LifecycleModule;
|
||||
import com.google.gerrit.lucene.LuceneAccountIndex;
|
||||
import com.google.gerrit.lucene.LuceneIndexModule.SingleVersionModule;
|
||||
import com.google.gerrit.server.config.GerritServerConfig;
|
||||
import com.google.gerrit.server.index.IndexConfig;
|
||||
import com.google.gerrit.server.index.IndexModule;
|
||||
import com.google.gerrit.server.index.account.AccountIndex;
|
||||
import com.google.gerrit.server.index.change.ChangeIndex;
|
||||
import com.google.inject.Provides;
|
||||
import com.google.inject.Singleton;
|
||||
import com.google.inject.assistedinject.FactoryModuleBuilder;
|
||||
|
||||
import org.eclipse.jgit.lib.Config;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class ElasticIndexModule extends LifecycleModule {
|
||||
private final int threads;
|
||||
private final Map<String, Integer> singleVersions;
|
||||
|
||||
public static ElasticIndexModule singleVersionWithExplicitVersions(
|
||||
Map<String, Integer> versions, int threads) {
|
||||
return new ElasticIndexModule(versions, threads);
|
||||
}
|
||||
|
||||
public static ElasticIndexModule latestVersionWithOnlineUpgrade() {
|
||||
return new ElasticIndexModule(null, 0);
|
||||
}
|
||||
|
||||
private ElasticIndexModule(Map<String, Integer> singleVersions, int threads) {
|
||||
this.singleVersions = singleVersions;
|
||||
this.threads = threads;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void configure() {
|
||||
install(
|
||||
new FactoryModuleBuilder()
|
||||
.implement(ChangeIndex.class, ElasticChangeIndex.class)
|
||||
.build(ChangeIndex.Factory.class));
|
||||
install(
|
||||
new FactoryModuleBuilder()
|
||||
// until we implement Elasticsearch index for accounts we need to
|
||||
// use Lucene to make all tests green and Gerrit server to work
|
||||
.implement(AccountIndex.class, LuceneAccountIndex.class)
|
||||
.build(AccountIndex.Factory.class));
|
||||
|
||||
install(new IndexModule(threads));
|
||||
install(new SingleVersionModule(singleVersions));
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
IndexConfig getIndexConfig(@GerritServerConfig Config cfg) {
|
||||
return IndexConfig.fromConfig(cfg);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright (C) 2016 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.elasticsearch;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
class ElasticMapping {
|
||||
static class Builder {
|
||||
private final ImmutableMap.Builder<String, FieldProperties> fields =
|
||||
new ImmutableMap.Builder<>();
|
||||
|
||||
MappingProperties build() {
|
||||
MappingProperties properties = new MappingProperties();
|
||||
properties.properties = fields.build();
|
||||
return properties;
|
||||
}
|
||||
|
||||
Builder addExactField(String name) {
|
||||
FieldProperties key = new FieldProperties("string");
|
||||
key.index = "not_analyzed";
|
||||
FieldProperties properties = new FieldProperties("string");
|
||||
properties.fields = ImmutableMap.of("key", key);
|
||||
fields.put(name, properties);
|
||||
return this;
|
||||
}
|
||||
|
||||
Builder addTimestamp(String name) {
|
||||
FieldProperties properties = new FieldProperties("date");
|
||||
properties.type = "date";
|
||||
properties.format = "dateOptionalTime";
|
||||
fields.put(name, properties);
|
||||
return this;
|
||||
}
|
||||
|
||||
Builder addNumber(String name) {
|
||||
fields.put(name, new FieldProperties("long"));
|
||||
return this;
|
||||
}
|
||||
|
||||
Builder addString(String name) {
|
||||
fields.put(name, new FieldProperties("string"));
|
||||
return this;
|
||||
}
|
||||
|
||||
Builder add(String name, String type) {
|
||||
fields.put(name, new FieldProperties(type));
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
static class MappingProperties {
|
||||
Map<String, FieldProperties> properties;
|
||||
}
|
||||
|
||||
static class FieldProperties {
|
||||
String type;
|
||||
String index;
|
||||
String format;
|
||||
Map<String, FieldProperties> fields;
|
||||
|
||||
FieldProperties(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright (C) 2014 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.elasticsearch;
|
||||
|
||||
import com.google.gerrit.server.index.FieldDef;
|
||||
import com.google.gerrit.server.index.FieldType;
|
||||
import com.google.gerrit.server.index.IndexPredicate;
|
||||
import com.google.gerrit.server.index.IntegerRangePredicate;
|
||||
import com.google.gerrit.server.index.RegexPredicate;
|
||||
import com.google.gerrit.server.index.TimestampRangePredicate;
|
||||
import com.google.gerrit.server.query.AndPredicate;
|
||||
import com.google.gerrit.server.query.NotPredicate;
|
||||
import com.google.gerrit.server.query.OrPredicate;
|
||||
import com.google.gerrit.server.query.Predicate;
|
||||
import com.google.gerrit.server.query.QueryParseException;
|
||||
import com.google.gerrit.server.query.change.AfterPredicate;
|
||||
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
public class ElasticQueryBuilder {
|
||||
|
||||
protected <T> QueryBuilder toQueryBuilder(Predicate<T> p)
|
||||
throws QueryParseException {
|
||||
if (p instanceof AndPredicate) {
|
||||
return and(p);
|
||||
} else if (p instanceof OrPredicate) {
|
||||
return or(p);
|
||||
} else if (p instanceof NotPredicate) {
|
||||
return not(p);
|
||||
} else if (p instanceof IndexPredicate) {
|
||||
return fieldQuery((IndexPredicate<T>) p);
|
||||
} else {
|
||||
throw new QueryParseException("cannot create query for index: " + p);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> BoolQueryBuilder and(Predicate<T> p)
|
||||
throws QueryParseException {
|
||||
try {
|
||||
BoolQueryBuilder b = QueryBuilders.boolQuery();
|
||||
for (Predicate<T> c : p.getChildren()) {
|
||||
b.must(toQueryBuilder(c));
|
||||
}
|
||||
return b;
|
||||
} catch (BooleanQuery.TooManyClauses e) {
|
||||
throw new QueryParseException("cannot create query for index: " + p, e);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> BoolQueryBuilder or(Predicate<T> p)
|
||||
throws QueryParseException {
|
||||
try {
|
||||
BoolQueryBuilder q = QueryBuilders.boolQuery();
|
||||
for (Predicate<T> c : p.getChildren()) {
|
||||
q.should(toQueryBuilder(c));
|
||||
}
|
||||
return q;
|
||||
} catch (BooleanQuery.TooManyClauses e) {
|
||||
throw new QueryParseException("cannot create query for index: " + p, e);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> QueryBuilder not(Predicate<T> p)
|
||||
throws QueryParseException {
|
||||
Predicate<T> n = p.getChild(0);
|
||||
if (n instanceof TimestampRangePredicate) {
|
||||
return notTimestamp((TimestampRangePredicate<T>) n);
|
||||
}
|
||||
|
||||
// Lucene does not support negation, start with all and subtract.
|
||||
BoolQueryBuilder q = QueryBuilders.boolQuery();
|
||||
q.must(QueryBuilders.matchAllQuery());
|
||||
q.mustNot(toQueryBuilder(n));
|
||||
return q;
|
||||
}
|
||||
|
||||
private <T> QueryBuilder fieldQuery(IndexPredicate<T> p)
|
||||
throws QueryParseException {
|
||||
FieldType<?> type = p.getType();
|
||||
FieldDef<?,?> field = p.getField();
|
||||
String name = field.getName();
|
||||
String value = p.getValue();
|
||||
|
||||
if (type == FieldType.INTEGER) {
|
||||
// QueryBuilder encodes integer fields as prefix coded bits,
|
||||
// which elasticsearch's queryString can't handle.
|
||||
// Create integer terms with string representations instead.
|
||||
return QueryBuilders.termQuery(name, value);
|
||||
} else if (type == FieldType.INTEGER_RANGE) {
|
||||
return intRangeQuery(p);
|
||||
} else if (type == FieldType.TIMESTAMP) {
|
||||
return timestampQuery(p);
|
||||
} else if (type == FieldType.EXACT) {
|
||||
return exactQuery(p);
|
||||
} else if (type == FieldType.PREFIX) {
|
||||
return QueryBuilders.matchPhrasePrefixQuery(name, value);
|
||||
} else if (type == FieldType.FULL_TEXT) {
|
||||
return QueryBuilders.matchPhraseQuery(name, value);
|
||||
} else {
|
||||
throw FieldType.badFieldType(p.getType());
|
||||
}
|
||||
}
|
||||
|
||||
private <T> QueryBuilder intRangeQuery(IndexPredicate<T> p)
|
||||
throws QueryParseException {
|
||||
if (p instanceof IntegerRangePredicate) {
|
||||
IntegerRangePredicate<T> r = (IntegerRangePredicate<T>) p;
|
||||
int minimum = r.getMinimumValue();
|
||||
int maximum = r.getMaximumValue();
|
||||
if (minimum == maximum) {
|
||||
// Just fall back to a standard integer query.
|
||||
return QueryBuilders.termQuery(p.getField().getName(), minimum);
|
||||
}
|
||||
return QueryBuilders.rangeQuery(p.getField().getName())
|
||||
.gte(minimum)
|
||||
.lte(maximum);
|
||||
}
|
||||
throw new QueryParseException("not an integer range: " + p);
|
||||
}
|
||||
|
||||
private <T> QueryBuilder notTimestamp(TimestampRangePredicate<T> r)
|
||||
throws QueryParseException {
|
||||
if (r.getMinTimestamp().getTime() == 0) {
|
||||
return QueryBuilders.rangeQuery(r.getField().getName())
|
||||
.gt(new DateTime(r.getMaxTimestamp().getTime()));
|
||||
}
|
||||
throw new QueryParseException("cannot negate: " + r);
|
||||
}
|
||||
|
||||
private <T> QueryBuilder timestampQuery(IndexPredicate<T> p)
|
||||
throws QueryParseException {
|
||||
if (p instanceof TimestampRangePredicate) {
|
||||
TimestampRangePredicate<T> r =
|
||||
(TimestampRangePredicate<T>) p;
|
||||
if (p instanceof AfterPredicate) {
|
||||
return QueryBuilders.rangeQuery(r.getField().getName())
|
||||
.gte(new DateTime(r.getMinTimestamp().getTime()));
|
||||
}
|
||||
return QueryBuilders.rangeQuery(r.getField().getName())
|
||||
.gte(new DateTime(r.getMinTimestamp().getTime()))
|
||||
.lte(new DateTime(r.getMaxTimestamp().getTime()));
|
||||
}
|
||||
throw new QueryParseException("not a timestamp: " + p);
|
||||
}
|
||||
|
||||
private <T> QueryBuilder exactQuery(IndexPredicate<T> p){
|
||||
String name = p.getField().getName();
|
||||
String value = p.getValue();
|
||||
|
||||
if (value.isEmpty()) {
|
||||
return new BoolQueryBuilder().mustNot(QueryBuilders.existsQuery(name));
|
||||
} else if (p instanceof RegexPredicate) {
|
||||
if (value.startsWith("^")) {
|
||||
value = value.substring(1);
|
||||
}
|
||||
if (value.endsWith("$") && !value.endsWith("\\$")
|
||||
&& !value.endsWith("\\\\$")) {
|
||||
value = value.substring(0, value.length() - 1);
|
||||
}
|
||||
return QueryBuilders.regexpQuery(name + ".key", value);
|
||||
} else {
|
||||
return QueryBuilders.termQuery(name + ".key", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user