opendev/gerrit
Code Issues Proposed changes

Add ElasticQuerySource to ElasticLuceneIndex

Browse Source 
This commit is a no-op change that reduces duplicate code in the ES
indices by creating a shared implementation of QuerySource.

Change-Id: Ibbb0058c3a0d9b99516ed86a342bf809148becbb
This commit is contained in:
Patrick Hiesel
2017-11-16 11:06:06 +01:00
parent 077c55edf9
commit 7e7103d9b8
5 changed files with 352 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@@ -23,15 +23,21 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Streams;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
@@ -41,24 +47,38 @@ import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.protobuf.ProtobufCodec;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.core.Bulk;
import io.searchbox.core.Delete;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.IndicesExists;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
import org.apache.commons.codec.binary.Base64;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private static final Logger log = LoggerFactory.getLogger(AbstractElasticIndex.class);
protected static <T> List<T> decodeProtos(
JsonObject doc, String fieldName, ProtobufCodec<T> codec) {
JsonArray field = doc.getAsJsonArray(fieldName);
@@ -158,7 +178,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected io.searchbox.core.Index insert(String type, V v) throws IOException {
String id = getId(v);
String doc = toDoc(v);
String doc = toDocument(v);
return new io.searchbox.core.Index.Builder(doc).index(indexName).type(type).id(id).build();
}
@@ -166,7 +186,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
return !(element instanceof String) || !((String) element).isEmpty();
}
private String toDoc(V v) throws IOException {
private String toDocument(V v) throws IOException {
XContentBuilder builder = jsonBuilder().startObject();
for (Values<V> values : schema.buildFields(v)) {
String name = values.getField().getName();
@@ -184,6 +204,8 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
return builder.endObject().string();
}
protected abstract V fromDocument(JsonObject doc, Set<String> fields);
protected FieldBundle toFieldBundle(JsonObject doc) {
Map<String, FieldDef<V, ?>> allFields = getSchema().getFields();
ListMultimap<String, Object> rawFields = ArrayListMultimap.create();
@@ -215,4 +237,95 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
return new FieldBundle(rawFields);
}
protected class ElasticQuerySource implements DataSource<V> {
private final QueryOptions opts;
private final Search search;
ElasticQuerySource(Predicate<V> p, QueryOptions opts, String type, Sort sort)
throws QueryParseException {
this(p, opts, ImmutableList.of(type), ImmutableList.of(sort));
}
ElasticQuerySource(
Predicate<V> p, QueryOptions opts, Collection<String> types, Collection<Sort> sorts)
throws QueryParseException {
this.opts = opts;
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(opts.fields()));
search =
new Search.Builder(searchSource.toString())
.addType(types)
.addSort(sorts)
.addIndex(indexName)
.build();
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<V> read() throws OrmException {
return readImpl((doc) -> AbstractElasticIndex.this.fromDocument(doc, opts.fields()));
}
@Override
public ResultSet<FieldBundle> readRaw() throws OrmException {
return readImpl(AbstractElasticIndex.this::toFieldBundle);
}
@Override
public String toString() {
return search.toString();
}
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) throws OrmException {
try {
List<T> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
T mapperResult = mapper.apply(json.get(i).getAsJsonObject());
if (mapperResult != null) {
results.add(mapperResult);
}
}
}
} else {
log.error(result.getErrorMessage());
}
final List<T> r = Collections.unmodifiableList(results);
return new ResultSet<T>() {
@Override
public Iterator<T> iterator() {
return r.iterator();
}
@Override
public List<T> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
}
}

129
java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@@ -16,14 +16,11 @@ package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.account.AccountField.ID;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Account;
@@ -34,31 +31,19 @@ import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountField;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
implements AccountIndex {
@@ -73,8 +58,6 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
static final String ACCOUNTS = "accounts";
static final String ACCOUNTS_PREFIX = ACCOUNTS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticAccountIndex.class);
private final AccountMapping mapping;
private final Provider<AccountCache> accountCache;
@@ -111,7 +94,9 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@Override
public DataSource<AccountState> getSource(Predicate<AccountState> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::accountFields), ACCOUNTS, sort);
}
@Override
@@ -130,104 +115,18 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
return as.getAccount().getId().toString();
}
private class QuerySource implements DataSource<AccountState> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<AccountState> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.accountFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(ACCOUNTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
@Override
protected AccountState fromDocument(JsonObject json, Set<String> fields) {
JsonElement source = json.get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<AccountState> read() throws OrmException {
return readImpl(this::toAccountState);
}
@Override
public ResultSet<FieldBundle> readRaw() throws OrmException {
return readImpl(ElasticAccountIndex.this::toFieldBundle);
}
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) throws OrmException {
try {
List<T> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
T mapperResult = mapper.apply(json.get(i).getAsJsonObject());
if (mapperResult != null) {
results.add(mapperResult);
}
}
}
} else {
log.error(result.getErrorMessage());
}
final List<T> r = Collections.unmodifiableList(results);
return new ResultSet<T>() {
@Override
public Iterator<T> iterator() {
return r.iterator();
}
@Override
public List<T> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public String toString() {
return search.toString();
}
private AccountState toAccountState(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
Account.Id id = new Account.Id(source.getAsJsonObject().get(ID.getName()).getAsInt());
// Use the AccountCache rather than depending on any stored fields in the
// document (of which there shouldn't be any). The most expensive part to
// compute anyway is the effective group IDs, and we don't have a good way
// to reindex when those change.
return accountCache.get().get(id);
}
Account.Id id = new Account.Id(source.getAsJsonObject().get(ID.getName()).getAsInt());
// Use the AccountCache rather than depending on any stored fields in the
// document (of which there shouldn't be any). The most expensive part to
// compute anyway is the effective group IDs, and we don't have a good way
// to reindex when those change.
return accountCache.get().get(id);
}
}

492
java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@@ -35,7 +35,7 @@ import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Account;
@@ -54,39 +54,28 @@ import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.project.SubmitRuleOptions;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import org.apache.commons.codec.binary.Base64;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Secondary index implementation using Elasticsearch. */
class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
implements ChangeIndex {
private static final Logger log = LoggerFactory.getLogger(ElasticChangeIndex.class);
static class ChangeMapping {
MappingProperties openChanges;
MappingProperties closedChanges;
@@ -155,7 +144,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, QueryOptions opts)
public DataSource getSource(Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
List<String> indexes = Lists.newArrayListWithCapacity(2);
@@ -165,7 +154,16 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(CLOSED_CHANGES);
}
return new QuerySource(indexes, p, opts);
List<Sort> sorts =
ImmutableList.of(
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
for (Sort sort : sorts) {
sort.setIgnoreUnmapped();
}
QueryOptions filteredOpts = opts.filterFields(IndexUtils::changeFields);
return new ElasticQuerySource(p, filteredOpts, indexes, sorts);
}
@Override
@@ -183,306 +181,210 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
return cd.getId().toString();
}
private class QuerySource implements ChangeDataSource {
private final Search search;
private final Set<String> fields;
@Override
protected ChangeData fromDocument(JsonObject json, Set<String> fields) {
JsonElement sourceElement = json.get("_source");
if (sourceElement == null) {
sourceElement = json.getAsJsonObject().get("fields");
}
JsonObject source = sourceElement.getAsJsonObject();
JsonElement c = source.get(ChangeField.CHANGE.getName());
QuerySource(List<String> types, Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
List<Sort> sorts =
ImmutableList.of(
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
for (Sort sort : sorts) {
sort.setIgnoreUnmapped();
if (c == null) {
int id = source.get(ChangeField.LEGACY_ID.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
String projectName = checkNotNull(source.get(ChangeField.PROJECT.getName()).getAsString());
return changeDataFactory.create(
db.get(), new Project.NameKey(projectName), new Change.Id(id));
}
ChangeData cd =
changeDataFactory.create(
db.get(), CHANGE_CODEC.decode(Base64.decodeBase64(c.getAsString())));
// Any decoding that is done here must also be done in {@link LuceneChangeIndex}.
// Patch sets.
cd.setPatchSets(decodeProtos(source, ChangeField.PATCH_SET.getName(), PATCH_SET_CODEC));
// Approvals.
if (source.get(ChangeField.APPROVAL.getName()) != null) {
cd.setCurrentApprovals(decodeProtos(source, ChangeField.APPROVAL.getName(), APPROVAL_CODEC));
} else if (fields.contains(ChangeField.APPROVAL.getName())) {
cd.setCurrentApprovals(Collections.emptyList());
}
// Added & Deleted.
JsonElement addedElement = source.get(ChangeField.ADDED.getName());
JsonElement deletedElement = source.get(ChangeField.DELETED.getName());
if (addedElement != null && deletedElement != null) {
// Changed lines.
int added = addedElement.getAsInt();
int deleted = deletedElement.getAsInt();
if (added != 0 && deleted != 0) {
cd.setChangedLines(added, deleted);
}
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.changeFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
search =
new Search.Builder(searchSource.toString())
.addType(types)
.addSort(sorts)
.addIndex(indexName)
.build();
}
@Override
public int getCardinality() {
return 10;
// Mergeable.
JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName());
if (mergeableElement != null) {
String mergeable = mergeableElement.getAsString();
if ("1".equals(mergeable)) {
cd.setMergeable(true);
} else if ("0".equals(mergeable)) {
cd.setMergeable(false);
}
}
@Override
public ResultSet<ChangeData> read() throws OrmException {
return readImpl(this::toChangeData);
}
@Override
public ResultSet<FieldBundle> readRaw() throws OrmException {
return readImpl(ElasticChangeIndex.this::toFieldBundle);
}
@Override
public boolean hasChange() {
return false;
}
@Override
public String toString() {
return search.toString();
}
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) throws OrmException {
try {
List<T> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
results.add(mapper.apply(json.get(i).getAsJsonObject()));
}
// Reviewed-by.
if (source.get(ChangeField.REVIEWEDBY.getName()) != null) {
JsonArray reviewedBy = source.get(ChangeField.REVIEWEDBY.getName()).getAsJsonArray();
if (reviewedBy.size() > 0) {
Set<Account.Id> accounts = Sets.newHashSetWithExpectedSize(reviewedBy.size());
for (int i = 0; i < reviewedBy.size(); i++) {
int aId = reviewedBy.get(i).getAsInt();
if (reviewedBy.size() == 1 && aId == ChangeField.NOT_REVIEWED) {
break;
}
} else {
log.error(result.getErrorMessage());
accounts.add(new Account.Id(aId));
}
final List<T> r = Collections.unmodifiableList(results);
return new ResultSet<T>() {
@Override
public Iterator<T> iterator() {
return r.iterator();
}
@Override
public List<T> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
cd.setReviewedBy(accounts);
}
} else if (fields.contains(ChangeField.REVIEWEDBY.getName())) {
cd.setReviewedBy(Collections.emptySet());
}
private ChangeData toChangeData(JsonElement json) {
JsonElement sourceElement = json.getAsJsonObject().get("_source");
if (sourceElement == null) {
sourceElement = json.getAsJsonObject().get("fields");
}
JsonObject source = sourceElement.getAsJsonObject();
JsonElement c = source.get(ChangeField.CHANGE.getName());
if (c == null) {
int id = source.get(ChangeField.LEGACY_ID.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
String projectName = checkNotNull(source.get(ChangeField.PROJECT.getName()).getAsString());
return changeDataFactory.create(
db.get(), new Project.NameKey(projectName), new Change.Id(id));
}
ChangeData cd =
changeDataFactory.create(
db.get(), CHANGE_CODEC.decode(Base64.decodeBase64(c.getAsString())));
// Any decoding that is done here must also be done in {@link LuceneChangeIndex}.
// Patch sets.
cd.setPatchSets(decodeProtos(source, ChangeField.PATCH_SET.getName(), PATCH_SET_CODEC));
// Approvals.
if (source.get(ChangeField.APPROVAL.getName()) != null) {
cd.setCurrentApprovals(
decodeProtos(source, ChangeField.APPROVAL.getName(), APPROVAL_CODEC));
} else if (fields.contains(ChangeField.APPROVAL.getName())) {
cd.setCurrentApprovals(Collections.emptyList());
}
// Added & Deleted.
JsonElement addedElement = source.get(ChangeField.ADDED.getName());
JsonElement deletedElement = source.get(ChangeField.DELETED.getName());
if (addedElement != null && deletedElement != null) {
// Changed lines.
int added = addedElement.getAsInt();
int deleted = deletedElement.getAsInt();
if (added != 0 && deleted != 0) {
cd.setChangedLines(added, deleted);
// Hashtag.
if (source.get(ChangeField.HASHTAG.getName()) != null) {
JsonArray hashtagArray = source.get(ChangeField.HASHTAG.getName()).getAsJsonArray();
if (hashtagArray.size() > 0) {
Set<String> hashtags = Sets.newHashSetWithExpectedSize(hashtagArray.size());
for (int i = 0; i < hashtagArray.size(); i++) {
hashtags.add(hashtagArray.get(i).getAsString());
}
cd.setHashtags(hashtags);
}
// Mergeable.
JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName());
if (mergeableElement != null) {
String mergeable = mergeableElement.getAsString();
if ("1".equals(mergeable)) {
cd.setMergeable(true);
} else if ("0".equals(mergeable)) {
cd.setMergeable(false);
}
}
// Reviewed-by.
if (source.get(ChangeField.REVIEWEDBY.getName()) != null) {
JsonArray reviewedBy = source.get(ChangeField.REVIEWEDBY.getName()).getAsJsonArray();
if (reviewedBy.size() > 0) {
Set<Account.Id> accounts = Sets.newHashSetWithExpectedSize(reviewedBy.size());
for (int i = 0; i < reviewedBy.size(); i++) {
int aId = reviewedBy.get(i).getAsInt();
if (reviewedBy.size() == 1 && aId == ChangeField.NOT_REVIEWED) {
break;
}
accounts.add(new Account.Id(aId));
}
cd.setReviewedBy(accounts);
}
} else if (fields.contains(ChangeField.REVIEWEDBY.getName())) {
cd.setReviewedBy(Collections.emptySet());
}
// Hashtag.
if (source.get(ChangeField.HASHTAG.getName()) != null) {
JsonArray hashtagArray = source.get(ChangeField.HASHTAG.getName()).getAsJsonArray();
if (hashtagArray.size() > 0) {
Set<String> hashtags = Sets.newHashSetWithExpectedSize(hashtagArray.size());
for (int i = 0; i < hashtagArray.size(); i++) {
hashtags.add(hashtagArray.get(i).getAsString());
}
cd.setHashtags(hashtags);
}
} else if (fields.contains(ChangeField.HASHTAG.getName())) {
cd.setHashtags(Collections.emptySet());
}
// Star.
if (source.get(ChangeField.STAR.getName()) != null) {
JsonArray starArray = source.get(ChangeField.STAR.getName()).getAsJsonArray();
if (starArray.size() > 0) {
ListMultimap<Account.Id, String> stars =
MultimapBuilder.hashKeys().arrayListValues().build();
for (int i = 0; i < starArray.size(); i++) {
StarredChangesUtil.StarField starField =
StarredChangesUtil.StarField.parse(starArray.get(i).getAsString());
stars.put(starField.accountId(), starField.label());
}
cd.setStars(stars);
}
} else if (fields.contains(ChangeField.STAR.getName())) {
cd.setStars(ImmutableListMultimap.of());
}
// Reviewer.
if (source.get(ChangeField.REVIEWER.getName()) != null) {
cd.setReviewers(
ChangeField.parseReviewerFieldValues(
FluentIterable.from(source.get(ChangeField.REVIEWER.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.REVIEWER.getName())) {
cd.setReviewers(ReviewerSet.empty());
}
// Reviewer-by-email.
if (source.get(ChangeField.REVIEWER_BY_EMAIL.getName()) != null) {
cd.setReviewersByEmail(
ChangeField.parseReviewerByEmailFieldValues(
FluentIterable.from(
source.get(ChangeField.REVIEWER_BY_EMAIL.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.REVIEWER_BY_EMAIL.getName())) {
cd.setReviewersByEmail(ReviewerByEmailSet.empty());
}
// Pending-reviewer.
if (source.get(ChangeField.PENDING_REVIEWER.getName()) != null) {
cd.setPendingReviewers(
ChangeField.parseReviewerFieldValues(
FluentIterable.from(
source.get(ChangeField.PENDING_REVIEWER.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.PENDING_REVIEWER.getName())) {
cd.setPendingReviewers(ReviewerSet.empty());
}
// Pending-reviewer-by-email.
if (source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()) != null) {
cd.setPendingReviewersByEmail(
ChangeField.parseReviewerByEmailFieldValues(
FluentIterable.from(
source
.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())
.getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())) {
cd.setPendingReviewersByEmail(ReviewerByEmailSet.empty());
}
// Stored-submit-record-strict.
decodeSubmitRecords(
source,
ChangeField.STORED_SUBMIT_RECORD_STRICT.getName(),
ChangeField.SUBMIT_RULE_OPTIONS_STRICT,
cd);
// Stored-submit-record-leniant.
decodeSubmitRecords(
source,
ChangeField.STORED_SUBMIT_RECORD_LENIENT.getName(),
ChangeField.SUBMIT_RULE_OPTIONS_LENIENT,
cd);
// Ref-state.
if (fields.contains(ChangeField.REF_STATE.getName())) {
cd.setRefStates(getByteArray(source, ChangeField.REF_STATE.getName()));
}
// Ref-state-pattern.
if (fields.contains(ChangeField.REF_STATE_PATTERN.getName())) {
cd.setRefStatePatterns(getByteArray(source, ChangeField.REF_STATE_PATTERN.getName()));
}
// Unresolved-comment-count.
decodeUnresolvedCommentCount(source, ChangeField.UNRESOLVED_COMMENT_COUNT.getName(), cd);
return cd;
} else if (fields.contains(ChangeField.HASHTAG.getName())) {
cd.setHashtags(Collections.emptySet());
}
private Iterable<byte[]> getByteArray(JsonObject source, String name) {
JsonElement element = source.get(name);
return element != null
? Iterables.transform(element.getAsJsonArray(), e -> Base64.decodeBase64(e.getAsString()))
: Collections.emptyList();
// Star.
if (source.get(ChangeField.STAR.getName()) != null) {
JsonArray starArray = source.get(ChangeField.STAR.getName()).getAsJsonArray();
if (starArray.size() > 0) {
ListMultimap<Account.Id, String> stars =
MultimapBuilder.hashKeys().arrayListValues().build();
for (int i = 0; i < starArray.size(); i++) {
StarredChangesUtil.StarField starField =
StarredChangesUtil.StarField.parse(starArray.get(i).getAsString());
stars.put(starField.accountId(), starField.label());
}
cd.setStars(stars);
}
} else if (fields.contains(ChangeField.STAR.getName())) {
cd.setStars(ImmutableListMultimap.of());
}
private void decodeSubmitRecords(
JsonObject doc, String fieldName, SubmitRuleOptions opts, ChangeData out) {
JsonArray records = doc.getAsJsonArray(fieldName);
if (records == null) {
return;
}
ChangeField.parseSubmitRecords(
FluentIterable.from(records)
.transform(i -> new String(decodeBase64(i.toString()), UTF_8))
.toList(),
opts,
out);
// Reviewer.
if (source.get(ChangeField.REVIEWER.getName()) != null) {
cd.setReviewers(
ChangeField.parseReviewerFieldValues(
FluentIterable.from(source.get(ChangeField.REVIEWER.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.REVIEWER.getName())) {
cd.setReviewers(ReviewerSet.empty());
}
private void decodeUnresolvedCommentCount(JsonObject doc, String fieldName, ChangeData out) {
JsonElement count = doc.get(fieldName);
if (count == null) {
return;
}
out.setUnresolvedCommentCount(count.getAsInt());
// Reviewer-by-email.
if (source.get(ChangeField.REVIEWER_BY_EMAIL.getName()) != null) {
cd.setReviewersByEmail(
ChangeField.parseReviewerByEmailFieldValues(
FluentIterable.from(
source.get(ChangeField.REVIEWER_BY_EMAIL.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.REVIEWER_BY_EMAIL.getName())) {
cd.setReviewersByEmail(ReviewerByEmailSet.empty());
}
// Pending-reviewer.
if (source.get(ChangeField.PENDING_REVIEWER.getName()) != null) {
cd.setPendingReviewers(
ChangeField.parseReviewerFieldValues(
FluentIterable.from(
source.get(ChangeField.PENDING_REVIEWER.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.PENDING_REVIEWER.getName())) {
cd.setPendingReviewers(ReviewerSet.empty());
}
// Pending-reviewer-by-email.
if (source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()) != null) {
cd.setPendingReviewersByEmail(
ChangeField.parseReviewerByEmailFieldValues(
FluentIterable.from(
source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())) {
cd.setPendingReviewersByEmail(ReviewerByEmailSet.empty());
}
// Stored-submit-record-strict.
decodeSubmitRecords(
source,
ChangeField.STORED_SUBMIT_RECORD_STRICT.getName(),
ChangeField.SUBMIT_RULE_OPTIONS_STRICT,
cd);
// Stored-submit-record-leniant.
decodeSubmitRecords(
source,
ChangeField.STORED_SUBMIT_RECORD_LENIENT.getName(),
ChangeField.SUBMIT_RULE_OPTIONS_LENIENT,
cd);
// Ref-state.
if (fields.contains(ChangeField.REF_STATE.getName())) {
cd.setRefStates(getByteArray(source, ChangeField.REF_STATE.getName()));
}
// Ref-state-pattern.
if (fields.contains(ChangeField.REF_STATE_PATTERN.getName())) {
cd.setRefStatePatterns(getByteArray(source, ChangeField.REF_STATE_PATTERN.getName()));
}
// Unresolved-comment-count.
decodeUnresolvedCommentCount(source, ChangeField.UNRESOLVED_COMMENT_COUNT.getName(), cd);
return cd;
}
private Iterable<byte[]> getByteArray(JsonObject source, String name) {
JsonElement element = source.get(name);
return element != null
? Iterables.transform(element.getAsJsonArray(), e -> Base64.decodeBase64(e.getAsString()))
: Collections.emptyList();
}
private void decodeSubmitRecords(
JsonObject doc, String fieldName, SubmitRuleOptions opts, ChangeData out) {
JsonArray records = doc.getAsJsonArray(fieldName);
if (records == null) {
return;
}
ChangeField.parseSubmitRecords(
FluentIterable.from(records)
.transform(i -> new String(decodeBase64(i.toString()), UTF_8))
.toList(),
opts,
out);
}
private void decodeUnresolvedCommentCount(JsonObject doc, String fieldName, ChangeData out) {
JsonElement count = doc.get(fieldName);
if (count == null) {
return;
}
out.setUnresolvedCommentCount(count.getAsInt());
}
}

130
java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@@ -14,14 +14,11 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.AccountGroup;
@@ -32,31 +29,18 @@ import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.group.GroupField;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, InternalGroup>
implements GroupIndex {
@@ -71,8 +55,6 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
static final String GROUPS = "groups";
static final String GROUPS_PREFIX = GROUPS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticGroupIndex.class);
private final GroupMapping mapping;
private final Provider<GroupCache> groupCache;
@@ -109,7 +91,9 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@Override
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
Sort sort = new Sort(GroupField.UUID.getName(), Sort.Sorting.ASC);
sort.setIgnoreUnmapped();
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), GROUPS, sort);
}
@Override
@@ -128,104 +112,18 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
return group.getGroupUUID().get();
}
private class QuerySource implements DataSource<InternalGroup> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<InternalGroup> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.groupFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(GroupField.UUID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(GROUPS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
@Override
protected InternalGroup fromDocument(JsonObject json, Set<String> fields) {
JsonElement source = json.get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<InternalGroup> read() throws OrmException {
return readImpl(this::toInternalGroup);
}
@Override
public ResultSet<FieldBundle> readRaw() throws OrmException {
return readImpl(ElasticGroupIndex.this::toFieldBundle);
}
@Override
public String toString() {
return search.toString();
}
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) throws OrmException {
try {
List<T> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
T mapperResult = mapper.apply(json.get(i).getAsJsonObject());
if (mapperResult != null) {
results.add(mapperResult);
}
}
}
} else {
log.error(result.getErrorMessage());
}
final List<T> r = Collections.unmodifiableList(results);
return new ResultSet<T>() {
@Override
public Iterator<T> iterator() {
return r.iterator();
}
@Override
public List<T> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
private InternalGroup toInternalGroup(JsonObject json) {
JsonElement source = json.get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
AccountGroup.UUID uuid =
new AccountGroup.UUID(
source.getAsJsonObject().get(GroupField.UUID.getName()).getAsString());
// Use the GroupCache rather than depending on any stored fields in the
// document (of which there shouldn't be any).
return groupCache.get().get(uuid).orElse(null);
}
AccountGroup.UUID uuid =
new AccountGroup.UUID(
source.getAsJsonObject().get(GroupField.UUID.getName()).getAsString());
// Use the GroupCache rather than depending on any stored fields in the
// document (of which there shouldn't be any).
return groupCache.get().get(uuid).orElse(null);
}
}

118
java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
View File

@@ -14,14 +14,11 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
@@ -32,30 +29,19 @@ import com.google.gerrit.server.index.project.ProjectField;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectData;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, ProjectData>
implements ProjectIndex {
@@ -70,8 +56,6 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
static final String PROJECTS = "projects";
static final String PROJECTS_PREFIX = PROJECTS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticProjectIndex.class);
private final ProjectMapping mapping;
private final Provider<ProjectCache> projectCache;
@@ -108,7 +92,9 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), PROJECTS, sort);
}
@Override
@@ -127,96 +113,16 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
return projectState.getProject().getName();
}
private class QuerySource implements DataSource<ProjectData> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<ProjectData> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.projectFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(PROJECTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
@Override
protected ProjectData fromDocument(JsonObject json, Set<String> fields) {
JsonElement source = json.get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<ProjectData> read() throws OrmException {
try {
List<ProjectData> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
results.add(toProjectData(json.get(i)));
}
}
} else {
log.error(result.getErrorMessage());
}
final List<ProjectData> r = Collections.unmodifiableList(results);
return new ResultSet<ProjectData>() {
@Override
public Iterator<ProjectData> iterator() {
return r.iterator();
}
@Override
public List<ProjectData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public ResultSet<FieldBundle> readRaw() throws OrmException {
// TOOD(hiesel): Make a generic implementation for Lucene/ES
throw new UnsupportedOperationException("not implemented");
}
@Override
public String toString() {
return search.toString();
}
private ProjectData toProjectData(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
Project.NameKey nameKey =
new Project.NameKey(
source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
return projectCache.get().get(nameKey).toProjectData();
}
Project.NameKey nameKey =
new Project.NameKey(
source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
return projectCache.get().get(nameKey).toProjectData();
}
}