opendev/gerrit
Code Issues Proposed changes

Dissolve gerrit-elasticsearch top-level directory

Browse Source 
Change-Id: Iede5fe438d0166b3c1c805be15f83f6099ca31f1
This commit is contained in:
David Ostrovsky
2017-08-23 00:53:24 +02:00
committed by Dave Borowitz
parent 54c8b0f00e
commit 32a3b1c706
22 changed files with 72 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@@ -1,174 +0,0 @@
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.util.stream.Collectors.toList;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gwtorm.protobuf.ProtobufCodec;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.core.Bulk;
import io.searchbox.core.Delete;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.IndicesExists;
import java.io.IOException;
import java.util.List;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.common.xcontent.XContentBuilder;
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected static <T> List<T> decodeProtos(
JsonObject doc, String fieldName, ProtobufCodec<T> codec) {
JsonArray field = doc.getAsJsonArray(fieldName);
if (field == null) {
return null;
}
return FluentIterable.from(field)
.transform(i -> codec.decode(decodeBase64(i.toString())))
.toList();
}
private final Schema<V> schema;
private final SitePaths sitePaths;
protected final String indexName;
protected final JestHttpClient client;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
AbstractElasticIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Schema<V> schema,
JestClientBuilder clientBuilder,
String indexName) {
this.sitePaths = sitePaths;
this.schema = schema;
this.gson = new GsonBuilder().setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES).create();
this.queryBuilder = new ElasticQueryBuilder();
this.indexName =
String.format(
"%s%s%04d",
Strings.nullToEmpty(cfg.getString("elasticsearch", null, "prefix")),
indexName,
schema.getVersion());
this.client = clientBuilder.build();
}
@Override
public Schema<V> getSchema() {
return schema;
}
@Override
public void close() {
client.shutdownClient();
}
@Override
public void markReady(boolean ready) throws IOException {
IndexUtils.setReady(sitePaths, indexName, schema.getVersion(), ready);
}
@Override
public void delete(K c) throws IOException {
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
throw new IOException(
String.format(
"Failed to delete change %s in index %s: %s",
c, indexName, result.getErrorMessage()));
}
}
@Override
public void deleteAll() throws IOException {
// Delete the index, if it exists.
JestResult result = client.execute(new IndicesExists.Builder(indexName).build());
if (result.isSucceeded()) {
result = client.execute(new DeleteIndex.Builder(indexName).build());
if (!result.isSucceeded()) {
throw new IOException(
String.format("Failed to delete index %s: %s", indexName, result.getErrorMessage()));
}
}
// Recreate the index.
result = client.execute(new CreateIndex.Builder(indexName).settings(getMappings()).build());
if (!result.isSucceeded()) {
String error =
String.format("Failed to create index %s: %s", indexName, result.getErrorMessage());
throw new IOException(error);
}
}
protected abstract Bulk.Builder addActions(Bulk.Builder builder, K c);
protected abstract String getMappings();
protected abstract String getId(V v);
protected Delete delete(String type, K c) {
String id = c.toString();
return new Delete.Builder(id).index(indexName).type(type).build();
}
protected io.searchbox.core.Index insert(String type, V v) throws IOException {
String id = getId(v);
String doc = toDoc(v);
return new io.searchbox.core.Index.Builder(doc).index(indexName).type(type).id(id).build();
}
private static boolean shouldAddElement(Object element) {
return !(element instanceof String) || !((String) element).isEmpty();
}
private String toDoc(V v) throws IOException {
XContentBuilder builder = jsonBuilder().startObject();
for (Values<V> values : schema.buildFields(v)) {
String name = values.getField().getName();
if (values.getField().isRepeatable()) {
builder.field(
name,
Streams.stream(values.getValues()).filter(e -> shouldAddElement(e)).collect(toList()));
} else {
Object element = Iterables.getOnlyElement(values.getValues(), "");
if (shouldAddElement(element)) {
builder.field(name, element);
}
}
}
return builder.endObject().string();
}
}

219
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@@ -1,219 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.account.AccountField.ID;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountField;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
implements AccountIndex {
static class AccountMapping {
MappingProperties accounts;
AccountMapping(Schema<AccountState> schema) {
this.accounts = ElasticMapping.createMapping(schema);
}
}
static final String ACCOUNTS = "accounts";
static final String ACCOUNTS_PREFIX = ACCOUNTS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticAccountIndex.class);
private final AccountMapping mapping;
private final Provider<AccountCache> accountCache;
@Inject
ElasticAccountIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
JestClientBuilder clientBuilder,
@Assisted Schema<AccountState> schema) {
super(cfg, sitePaths, schema, clientBuilder, ACCOUNTS_PREFIX);
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema);
}
@Override
public void replace(AccountState as) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(ACCOUNTS)
.addAction(insert(ACCOUNTS, as))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
throw new IOException(
String.format(
"Failed to replace account %s in index %s: %s",
as.getAccount().getId(), indexName, result.getErrorMessage()));
}
}
@Override
public DataSource<AccountState> getSource(Predicate<AccountState> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
}
@Override
protected Builder addActions(Builder builder, Account.Id c) {
return builder.addAction(delete(ACCOUNTS, c));
}
@Override
protected String getMappings() {
ImmutableMap<String, AccountMapping> mappings = ImmutableMap.of("mappings", mapping);
return gson.toJson(mappings);
}
@Override
protected String getId(AccountState as) {
return as.getAccount().getId().toString();
}
private class QuerySource implements DataSource<AccountState> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<AccountState> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.accountFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(ACCOUNTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<AccountState> read() throws OrmException {
try {
List<AccountState> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
results.add(toAccountState(json.get(i)));
}
}
} else {
log.error(result.getErrorMessage());
}
final List<AccountState> r = Collections.unmodifiableList(results);
return new ResultSet<AccountState>() {
@Override
public Iterator<AccountState> iterator() {
return r.iterator();
}
@Override
public List<AccountState> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public String toString() {
return search.toString();
}
private AccountState toAccountState(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
Account.Id id = new Account.Id(source.getAsJsonObject().get(ID.getName()).getAsInt());
// Use the AccountCache rather than depending on any stored fields in the
// document (of which there shouldn't be any). The most expensive part to
// compute anyway is the effective group IDs, and we don't have a good way
// to reindex when those change.
return accountCache.get().get(id);
}
}
}

426
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@@ -1,426 +0,0 @@
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gerrit.server.index.change.ChangeField.APPROVAL_CODEC;
import static com.google.gerrit.server.index.change.ChangeField.CHANGE_CODEC;
import static com.google.gerrit.server.index.change.ChangeField.PATCH_SET_CODEC;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.CLOSED_STATUSES;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.OPEN_STATUSES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Change.Id;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.ReviewerByEmailSet;
import com.google.gerrit.server.ReviewerSet;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.project.SubmitRuleOptions;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Secondary index implementation using Elasticsearch. */
class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
implements ChangeIndex {
private static final Logger log = LoggerFactory.getLogger(ElasticChangeIndex.class);
static class ChangeMapping {
MappingProperties openChanges;
MappingProperties closedChanges;
ChangeMapping(Schema<ChangeData> schema) {
MappingProperties mapping = ElasticMapping.createMapping(schema);
this.openChanges = mapping;
this.closedChanges = mapping;
}
}
static final String CHANGES_PREFIX = "changes_";
static final String OPEN_CHANGES = "open_changes";
static final String CLOSED_CHANGES = "closed_changes";
private final ChangeMapping mapping;
private final Provider<ReviewDb> db;
private final ChangeData.Factory changeDataFactory;
@Inject
ElasticChangeIndex(
@GerritServerConfig Config cfg,
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
SitePaths sitePaths,
JestClientBuilder clientBuilder,
@Assisted Schema<ChangeData> schema) {
super(cfg, sitePaths, schema, clientBuilder, CHANGES_PREFIX);
this.db = db;
this.changeDataFactory = changeDataFactory;
mapping = new ChangeMapping(schema);
}
@Override
public void replace(ChangeData cd) throws IOException {
String deleteIndex;
String insertIndex;
try {
if (cd.change().getStatus().isOpen()) {
insertIndex = OPEN_CHANGES;
deleteIndex = CLOSED_CHANGES;
} else {
insertIndex = CLOSED_CHANGES;
deleteIndex = OPEN_CHANGES;
}
} catch (OrmException e) {
throw new IOException(e);
}
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType("changes")
.addAction(insert(insertIndex, cd))
.addAction(delete(deleteIndex, cd.getId()))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
throw new IOException(
String.format(
"Failed to replace change %s in index %s: %s",
cd.getId(), indexName, result.getErrorMessage()));
}
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
List<String> indexes = Lists.newArrayListWithCapacity(2);
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(OPEN_CHANGES);
}
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(CLOSED_CHANGES);
}
return new QuerySource(indexes, p, opts);
}
@Override
protected Builder addActions(Builder builder, Id c) {
return builder.addAction(delete(OPEN_CHANGES, c)).addAction(delete(OPEN_CHANGES, c));
}
@Override
protected String getMappings() {
return gson.toJson(ImmutableMap.of("mappings", mapping));
}
@Override
protected String getId(ChangeData cd) {
return cd.getId().toString();
}
private class QuerySource implements ChangeDataSource {
private final Search search;
private final Set<String> fields;
QuerySource(List<String> types, Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
List<Sort> sorts =
ImmutableList.of(
new Sort(ChangeField.UPDATED.getName(), Sorting.DESC),
new Sort(ChangeField.LEGACY_ID.getName(), Sorting.DESC));
for (Sort sort : sorts) {
sort.setIgnoreUnmapped();
}
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.changeFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
search =
new Search.Builder(searchSource.toString())
.addType(types)
.addSort(sorts)
.addIndex(indexName)
.build();
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<ChangeData> read() throws OrmException {
try {
List<ChangeData> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
results.add(toChangeData(json.get(i)));
}
}
} else {
log.error(result.getErrorMessage());
}
final List<ChangeData> r = Collections.unmodifiableList(results);
return new ResultSet<ChangeData>() {
@Override
public Iterator<ChangeData> iterator() {
return r.iterator();
}
@Override
public List<ChangeData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public boolean hasChange() {
return false;
}
@Override
public String toString() {
return search.toString();
}
private ChangeData toChangeData(JsonElement json) {
JsonElement sourceElement = json.getAsJsonObject().get("_source");
if (sourceElement == null) {
sourceElement = json.getAsJsonObject().get("fields");
}
JsonObject source = sourceElement.getAsJsonObject();
JsonElement c = source.get(ChangeField.CHANGE.getName());
if (c == null) {
int id = source.get(ChangeField.LEGACY_ID.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
String projectName = checkNotNull(source.get(ChangeField.PROJECT.getName()).getAsString());
return changeDataFactory.create(
db.get(), new Project.NameKey(projectName), new Change.Id(id));
}
ChangeData cd =
changeDataFactory.create(
db.get(), CHANGE_CODEC.decode(Base64.decodeBase64(c.getAsString())));
// Patch sets.
cd.setPatchSets(decodeProtos(source, ChangeField.PATCH_SET.getName(), PATCH_SET_CODEC));
// Approvals.
if (source.get(ChangeField.APPROVAL.getName()) != null) {
cd.setCurrentApprovals(
decodeProtos(source, ChangeField.APPROVAL.getName(), APPROVAL_CODEC));
} else if (fields.contains(ChangeField.APPROVAL.getName())) {
cd.setCurrentApprovals(Collections.emptyList());
}
JsonElement addedElement = source.get(ChangeField.ADDED.getName());
JsonElement deletedElement = source.get(ChangeField.DELETED.getName());
if (addedElement != null && deletedElement != null) {
// Changed lines.
int added = addedElement.getAsInt();
int deleted = deletedElement.getAsInt();
if (added != 0 && deleted != 0) {
cd.setChangedLines(added, deleted);
}
}
// Mergeable.
JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName());
if (mergeableElement != null) {
String mergeable = mergeableElement.getAsString();
if ("1".equals(mergeable)) {
cd.setMergeable(true);
} else if ("0".equals(mergeable)) {
cd.setMergeable(false);
}
}
// Reviewed-by.
if (source.get(ChangeField.REVIEWEDBY.getName()) != null) {
JsonArray reviewedBy = source.get(ChangeField.REVIEWEDBY.getName()).getAsJsonArray();
if (reviewedBy.size() > 0) {
Set<Account.Id> accounts = Sets.newHashSetWithExpectedSize(reviewedBy.size());
for (int i = 0; i < reviewedBy.size(); i++) {
int aId = reviewedBy.get(i).getAsInt();
if (reviewedBy.size() == 1 && aId == ChangeField.NOT_REVIEWED) {
break;
}
accounts.add(new Account.Id(aId));
}
cd.setReviewedBy(accounts);
}
} else if (fields.contains(ChangeField.REVIEWEDBY.getName())) {
cd.setReviewedBy(Collections.emptySet());
}
if (source.get(ChangeField.REVIEWER.getName()) != null) {
cd.setReviewers(
ChangeField.parseReviewerFieldValues(
FluentIterable.from(source.get(ChangeField.REVIEWER.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.REVIEWER.getName())) {
cd.setReviewers(ReviewerSet.empty());
}
if (source.get(ChangeField.REVIEWER_BY_EMAIL.getName()) != null) {
cd.setReviewersByEmail(
ChangeField.parseReviewerByEmailFieldValues(
FluentIterable.from(
source.get(ChangeField.REVIEWER_BY_EMAIL.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.REVIEWER_BY_EMAIL.getName())) {
cd.setReviewersByEmail(ReviewerByEmailSet.empty());
}
if (source.get(ChangeField.PENDING_REVIEWER.getName()) != null) {
cd.setPendingReviewers(
ChangeField.parseReviewerFieldValues(
FluentIterable.from(
source.get(ChangeField.PENDING_REVIEWER.getName()).getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.PENDING_REVIEWER.getName())) {
cd.setPendingReviewers(ReviewerSet.empty());
}
if (source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()) != null) {
cd.setPendingReviewersByEmail(
ChangeField.parseReviewerByEmailFieldValues(
FluentIterable.from(
source
.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())
.getAsJsonArray())
.transform(JsonElement::getAsString)));
} else if (fields.contains(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())) {
cd.setPendingReviewersByEmail(ReviewerByEmailSet.empty());
}
decodeSubmitRecords(
source,
ChangeField.STORED_SUBMIT_RECORD_STRICT.getName(),
ChangeField.SUBMIT_RULE_OPTIONS_STRICT,
cd);
decodeSubmitRecords(
source,
ChangeField.STORED_SUBMIT_RECORD_LENIENT.getName(),
ChangeField.SUBMIT_RULE_OPTIONS_LENIENT,
cd);
decodeUnresolvedCommentCount(source, ChangeField.UNRESOLVED_COMMENT_COUNT.getName(), cd);
if (fields.contains(ChangeField.REF_STATE.getName())) {
cd.setRefStates(getByteArray(source, ChangeField.REF_STATE.getName()));
}
if (fields.contains(ChangeField.REF_STATE_PATTERN.getName())) {
cd.setRefStatePatterns(getByteArray(source, ChangeField.REF_STATE_PATTERN.getName()));
}
return cd;
}
private Iterable<byte[]> getByteArray(JsonObject source, String name) {
JsonElement element = source.get(name);
return element != null
? Iterables.transform(element.getAsJsonArray(), e -> Base64.decodeBase64(e.getAsString()))
: Collections.emptyList();
}
private void decodeSubmitRecords(
JsonObject doc, String fieldName, SubmitRuleOptions opts, ChangeData out) {
JsonArray records = doc.getAsJsonArray(fieldName);
if (records == null) {
return;
}
ChangeField.parseSubmitRecords(
FluentIterable.from(records)
.transform(i -> new String(decodeBase64(i.toString()), UTF_8))
.toList(),
opts,
out);
}
private void decodeUnresolvedCommentCount(JsonObject doc, String fieldName, ChangeData out) {
JsonElement count = doc.get(fieldName);
if (count == null) {
return;
}
out.setUnresolvedCommentCount(count.getAsInt());
}
}
}

92
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
View File

@@ -1,92 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.base.MoreObjects;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.lib.Config;
@Singleton
class ElasticConfiguration {
private static final String DEFAULT_HOST = "localhost";
private static final String DEFAULT_PORT = "9200";
private static final String DEFAULT_PROTOCOL = "http";
final List<String> urls;
final String username;
final String password;
final boolean requestCompression;
final long connectionTimeout;
final long maxConnectionIdleTime;
final TimeUnit maxConnectionIdleUnit = TimeUnit.MILLISECONDS;
final int maxTotalConnection;
final int readTimeout;
@Inject
ElasticConfiguration(@GerritServerConfig Config cfg) {
this.username = cfg.getString("elasticsearch", null, "username");
this.password = cfg.getString("elasticsearch", null, "password");
this.requestCompression = cfg.getBoolean("elasticsearch", null, "requestCompression", false);
this.connectionTimeout =
cfg.getTimeUnit("elasticsearch", null, "connectionTimeout", 3000, TimeUnit.MILLISECONDS);
this.maxConnectionIdleTime =
cfg.getTimeUnit(
"elasticsearch", null, "maxConnectionIdleTime", 3000, TimeUnit.MILLISECONDS);
this.maxTotalConnection = cfg.getInt("elasticsearch", null, "maxTotalConnection", 1);
this.readTimeout =
(int) cfg.getTimeUnit("elasticsearch", null, "readTimeout", 3000, TimeUnit.MICROSECONDS);
Set<String> subsections = cfg.getSubsections("elasticsearch");
if (subsections.isEmpty()) {
this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT));
} else {
this.urls = new ArrayList<>(subsections.size());
for (String subsection : subsections) {
String port = getString(cfg, subsection, "port", DEFAULT_PORT);
String host = getString(cfg, subsection, "hostname", DEFAULT_HOST);
String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL);
this.urls.add(buildUrl(protocol, host, port));
}
}
}
private String getString(Config cfg, String subsection, String name, String defaultValue) {
return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
}
private String buildUrl(String protocol, String hostname, String port) {
try {
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
} catch (MalformedURLException | NumberFormatException e) {
throw new RuntimeException(
"Cannot build url to Elasticsearch from values: protocol="
+ protocol
+ " hostname="
+ hostname
+ " port="
+ port,
e);
}
}
}

219
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@@ -1,219 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.server.account.GroupCache;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.group.GroupField;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, InternalGroup>
implements GroupIndex {
static class GroupMapping {
MappingProperties groups;
GroupMapping(Schema<InternalGroup> schema) {
this.groups = ElasticMapping.createMapping(schema);
}
}
static final String GROUPS = "groups";
static final String GROUPS_PREFIX = GROUPS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticGroupIndex.class);
private final GroupMapping mapping;
private final Provider<GroupCache> groupCache;
@Inject
ElasticGroupIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
JestClientBuilder clientBuilder,
@Assisted Schema<InternalGroup> schema) {
super(cfg, sitePaths, schema, clientBuilder, GROUPS_PREFIX);
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema);
}
@Override
public void replace(InternalGroup group) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(GROUPS)
.addAction(insert(GROUPS, group))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
throw new IOException(
String.format(
"Failed to replace group %s in index %s: %s",
group.getGroupUUID().get(), indexName, result.getErrorMessage()));
}
}
@Override
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
}
@Override
protected Builder addActions(Builder builder, AccountGroup.UUID c) {
return builder.addAction(delete(GROUPS, c));
}
@Override
protected String getMappings() {
ImmutableMap<String, GroupMapping> mappings = ImmutableMap.of("mappings", mapping);
return gson.toJson(mappings);
}
@Override
protected String getId(InternalGroup group) {
return group.getGroupUUID().get();
}
private class QuerySource implements DataSource<InternalGroup> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<InternalGroup> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.groupFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(GroupField.UUID.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(GROUPS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<InternalGroup> read() throws OrmException {
try {
List<InternalGroup> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
Optional<InternalGroup> internalGroup = toInternalGroup(json.get(i));
internalGroup.ifPresent(results::add);
}
}
} else {
log.error(result.getErrorMessage());
}
final List<InternalGroup> r = Collections.unmodifiableList(results);
return new ResultSet<InternalGroup>() {
@Override
public Iterator<InternalGroup> iterator() {
return r.iterator();
}
@Override
public List<InternalGroup> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public String toString() {
return search.toString();
}
private Optional<InternalGroup> toInternalGroup(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
AccountGroup.UUID uuid =
new AccountGroup.UUID(
source.getAsJsonObject().get(GroupField.UUID.getName()).getAsString());
// Use the GroupCache rather than depending on any stored fields in the
// document (of which there shouldn't be any).
return groupCache.get().get(uuid);
}
}
}

108
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java
View File

@@ -1,108 +0,0 @@
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.OnlineUpgrader;
import com.google.gerrit.server.index.SingleVersionModule;
import com.google.gerrit.server.index.VersionManager;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import java.util.Map;
import org.eclipse.jgit.lib.Config;
public class ElasticIndexModule extends AbstractModule {
public static ElasticIndexModule singleVersionWithExplicitVersions(
Map<String, Integer> versions, int threads) {
return new ElasticIndexModule(versions, threads, false);
}
public static ElasticIndexModule latestVersionWithOnlineUpgrade() {
return new ElasticIndexModule(null, 0, true);
}
public static ElasticIndexModule latestVersionWithoutOnlineUpgrade() {
return new ElasticIndexModule(null, 0, false);
}
private final Map<String, Integer> singleVersions;
private final int threads;
private final boolean onlineUpgrade;
private ElasticIndexModule(
Map<String, Integer> singleVersions, int threads, boolean onlineUpgrade) {
if (singleVersions != null) {
checkArgument(!onlineUpgrade, "online upgrade is incompatible with single version map");
}
this.singleVersions = singleVersions;
this.threads = threads;
this.onlineUpgrade = onlineUpgrade;
}
@Override
protected void configure() {
install(
new FactoryModuleBuilder()
.implement(AccountIndex.class, ElasticAccountIndex.class)
.build(AccountIndex.Factory.class));
install(
new FactoryModuleBuilder()
.implement(ChangeIndex.class, ElasticChangeIndex.class)
.build(ChangeIndex.Factory.class));
install(
new FactoryModuleBuilder()
.implement(GroupIndex.class, ElasticGroupIndex.class)
.build(GroupIndex.Factory.class));
install(
new FactoryModuleBuilder()
.implement(ProjectIndex.class, ElasticProjectIndex.class)
.build(ProjectIndex.Factory.class));
install(new IndexModule(threads));
if (singleVersions == null) {
install(new MultiVersionModule());
} else {
install(new SingleVersionModule(singleVersions));
}
}
@Provides
@Singleton
IndexConfig getIndexConfig(@GerritServerConfig Config cfg) {
return IndexConfig.fromConfig(cfg).separateChangeSubIndexes(true).build();
}
private class MultiVersionModule extends LifecycleModule {
@Override
public void configure() {
bind(VersionManager.class).to(ElasticVersionManager.class);
listener().to(ElasticVersionManager.class);
if (onlineUpgrade) {
listener().to(OnlineUpgrader.class);
}
}
}
}

52
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
View File

@@ -1,52 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestResult;
import io.searchbox.client.http.JestHttpClient;
import io.searchbox.indices.aliases.GetAliases;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
@Singleton
class ElasticIndexVersionDiscovery {
private final JestHttpClient client;
@Inject
ElasticIndexVersionDiscovery(JestClientBuilder clientBuilder) {
this.client = clientBuilder.build();
}
List<String> discover(String prefix, String indexName) throws IOException {
String name = prefix + indexName + "_";
JestResult result = client.execute(new GetAliases.Builder().addIndex(name + "*").build());
if (result.isSucceeded()) {
JsonObject object = result.getJsonObject().getAsJsonObject();
List<String> versions = new ArrayList<>(object.size());
for (Entry<String, JsonElement> entry : object.entrySet()) {
versions.add(entry.getKey().replace(name, ""));
}
return versions;
}
return Collections.emptyList();
}
}

105
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
View File

@@ -1,105 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Schema;
import java.util.Map;
class ElasticMapping {
static MappingProperties createMapping(Schema<?> schema) {
ElasticMapping.Builder mapping = new ElasticMapping.Builder();
for (FieldDef<?, ?> field : schema.getFields().values()) {
String name = field.getName();
FieldType<?> fieldType = field.getType();
if (fieldType == FieldType.EXACT) {
mapping.addExactField(name);
} else if (fieldType == FieldType.TIMESTAMP) {
mapping.addTimestamp(name);
} else if (fieldType == FieldType.INTEGER
|| fieldType == FieldType.INTEGER_RANGE
|| fieldType == FieldType.LONG) {
mapping.addNumber(name);
} else if (fieldType == FieldType.PREFIX
|| fieldType == FieldType.FULL_TEXT
|| fieldType == FieldType.STORED_ONLY) {
mapping.addString(name);
} else {
throw new IllegalStateException("Unsupported field type: " + fieldType.getName());
}
}
return mapping.build();
}
static class Builder {
private final ImmutableMap.Builder<String, FieldProperties> fields =
new ImmutableMap.Builder<>();
MappingProperties build() {
MappingProperties properties = new MappingProperties();
properties.properties = fields.build();
return properties;
}
Builder addExactField(String name) {
FieldProperties key = new FieldProperties("string");
key.index = "not_analyzed";
FieldProperties properties = new FieldProperties("string");
properties.fields = ImmutableMap.of("key", key);
fields.put(name, properties);
return this;
}
Builder addTimestamp(String name) {
FieldProperties properties = new FieldProperties("date");
properties.type = "date";
properties.format = "dateOptionalTime";
fields.put(name, properties);
return this;
}
Builder addNumber(String name) {
fields.put(name, new FieldProperties("long"));
return this;
}
Builder addString(String name) {
fields.put(name, new FieldProperties("string"));
return this;
}
Builder add(String name, String type) {
fields.put(name, new FieldProperties(type));
return this;
}
}
static class MappingProperties {
Map<String, FieldProperties> properties;
}
static class FieldProperties {
String type;
String index;
String format;
Map<String, FieldProperties> fields;
FieldProperties(String type) {
this.type = type;
}
}
}

215
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
View File

@@ -1,215 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.project.ProjectField;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectData;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, ProjectData>
implements ProjectIndex {
static class ProjectMapping {
MappingProperties projects;
ProjectMapping(Schema<ProjectData> schema) {
this.projects = ElasticMapping.createMapping(schema);
}
}
static final String PROJECTS = "projects";
static final String PROJECTS_PREFIX = PROJECTS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticProjectIndex.class);
private final ProjectMapping mapping;
private final Provider<ProjectCache> projectCache;
@Inject
ElasticProjectIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
JestClientBuilder clientBuilder,
@Assisted Schema<ProjectData> schema) {
super(cfg, sitePaths, schema, clientBuilder, PROJECTS_PREFIX);
this.projectCache = projectCache;
this.mapping = new ProjectMapping(schema);
}
@Override
public void replace(ProjectData projectState) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(PROJECTS)
.addAction(insert(PROJECTS, projectState))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
throw new IOException(
String.format(
"Failed to replace project %s in index %s: %s",
projectState.getProject().getName(), indexName, result.getErrorMessage()));
}
}
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
}
@Override
protected Builder addActions(Builder builder, Project.NameKey nameKey) {
return builder.addAction(delete(PROJECTS, nameKey));
}
@Override
protected String getMappings() {
ImmutableMap<String, ProjectMapping> mappings = ImmutableMap.of("mappings", mapping);
return gson.toJson(mappings);
}
@Override
protected String getId(ProjectData projectState) {
return projectState.getProject().getName();
}
private class QuerySource implements DataSource<ProjectData> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<ProjectData> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.projectFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(PROJECTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<ProjectData> read() throws OrmException {
try {
List<ProjectData> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
results.add(toProjectData(json.get(i)));
}
}
} else {
log.error(result.getErrorMessage());
}
final List<ProjectData> r = Collections.unmodifiableList(results);
return new ResultSet<ProjectData>() {
@Override
public Iterator<ProjectData> iterator() {
return r.iterator();
}
@Override
public List<ProjectData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public String toString() {
return search.toString();
}
private ProjectData toProjectData(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
Project.NameKey nameKey =
new Project.NameKey(
source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
return projectCache.get().get(nameKey).toProjectData();
}
}
}

168
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
View File

@@ -1,168 +0,0 @@
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.query.AndPredicate;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.IntegerRangePredicate;
import com.google.gerrit.index.query.NotPredicate;
import com.google.gerrit.index.query.OrPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.index.query.RegexPredicate;
import com.google.gerrit.index.query.TimestampRangePredicate;
import com.google.gerrit.server.query.change.AfterPredicate;
import java.time.Instant;
import org.apache.lucene.search.BooleanQuery;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
public class ElasticQueryBuilder {
protected <T> QueryBuilder toQueryBuilder(Predicate<T> p) throws QueryParseException {
if (p instanceof AndPredicate) {
return and(p);
} else if (p instanceof OrPredicate) {
return or(p);
} else if (p instanceof NotPredicate) {
return not(p);
} else if (p instanceof IndexPredicate) {
return fieldQuery((IndexPredicate<T>) p);
} else {
throw new QueryParseException("cannot create query for index: " + p);
}
}
private <T> BoolQueryBuilder and(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
b.must(toQueryBuilder(c));
}
return b;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
}
}
private <T> BoolQueryBuilder or(Predicate<T> p) throws QueryParseException {
try {
BoolQueryBuilder q = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
q.should(toQueryBuilder(c));
}
return q;
} catch (BooleanQuery.TooManyClauses e) {
throw new QueryParseException("cannot create query for index: " + p, e);
}
}
private <T> QueryBuilder not(Predicate<T> p) throws QueryParseException {
Predicate<T> n = p.getChild(0);
if (n instanceof TimestampRangePredicate) {
return notTimestamp((TimestampRangePredicate<T>) n);
}
// Lucene does not support negation, start with all and subtract.
BoolQueryBuilder q = QueryBuilders.boolQuery();
q.must(QueryBuilders.matchAllQuery());
q.mustNot(toQueryBuilder(n));
return q;
}
private <T> QueryBuilder fieldQuery(IndexPredicate<T> p) throws QueryParseException {
FieldType<?> type = p.getType();
FieldDef<?, ?> field = p.getField();
String name = field.getName();
String value = p.getValue();
if (type == FieldType.INTEGER) {
// QueryBuilder encodes integer fields as prefix coded bits,
// which elasticsearch's queryString can't handle.
// Create integer terms with string representations instead.
return QueryBuilders.termQuery(name, value);
} else if (type == FieldType.INTEGER_RANGE) {
return intRangeQuery(p);
} else if (type == FieldType.TIMESTAMP) {
return timestampQuery(p);
} else if (type == FieldType.EXACT) {
return exactQuery(p);
} else if (type == FieldType.PREFIX) {
return QueryBuilders.matchPhrasePrefixQuery(name, value);
} else if (type == FieldType.FULL_TEXT) {
return QueryBuilders.matchPhraseQuery(name, value);
} else {
throw FieldType.badFieldType(p.getType());
}
}
private <T> QueryBuilder intRangeQuery(IndexPredicate<T> p) throws QueryParseException {
if (p instanceof IntegerRangePredicate) {
IntegerRangePredicate<T> r = (IntegerRangePredicate<T>) p;
int minimum = r.getMinimumValue();
int maximum = r.getMaximumValue();
if (minimum == maximum) {
// Just fall back to a standard integer query.
return QueryBuilders.termQuery(p.getField().getName(), minimum);
}
return QueryBuilders.rangeQuery(p.getField().getName()).gte(minimum).lte(maximum);
}
throw new QueryParseException("not an integer range: " + p);
}
private <T> QueryBuilder notTimestamp(TimestampRangePredicate<T> r) throws QueryParseException {
if (r.getMinTimestamp().getTime() == 0) {
return QueryBuilders.rangeQuery(r.getField().getName())
.gt(Instant.ofEpochMilli(r.getMaxTimestamp().getTime()));
}
throw new QueryParseException("cannot negate: " + r);
}
private <T> QueryBuilder timestampQuery(IndexPredicate<T> p) throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<T> r = (TimestampRangePredicate<T>) p;
if (p instanceof AfterPredicate) {
return QueryBuilders.rangeQuery(r.getField().getName())
.gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime()));
}
return QueryBuilders.rangeQuery(r.getField().getName())
.gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime()))
.lte(Instant.ofEpochMilli(r.getMaxTimestamp().getTime()));
}
throw new QueryParseException("not a timestamp: " + p);
}
private <T> QueryBuilder exactQuery(IndexPredicate<T> p) {
String name = p.getField().getName();
String value = p.getValue();
if (value.isEmpty()) {
return new BoolQueryBuilder().mustNot(QueryBuilders.existsQuery(name));
} else if (p instanceof RegexPredicate) {
if (value.startsWith("^")) {
value = value.substring(1);
}
if (value.endsWith("$") && !value.endsWith("\\$") && !value.endsWith("\\\\$")) {
value = value.substring(0, value.length() - 1);
}
return QueryBuilders.regexpQuery(name + ".key", value);
} else {
return QueryBuilders.termQuery(name + ".key", value);
}
}
}

86
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java
View File

@@ -1,86 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.base.MoreObjects;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.index.Schema;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.GerritIndexStatus;
import com.google.gerrit.server.index.OnlineUpgradeListener;
import com.google.gerrit.server.index.VersionManager;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.Collection;
import java.util.TreeMap;
import org.eclipse.jgit.lib.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class ElasticVersionManager extends VersionManager {
private static final Logger log = LoggerFactory.getLogger(ElasticVersionManager.class);
private final String prefix;
private final ElasticIndexVersionDiscovery versionDiscovery;
@Inject
ElasticVersionManager(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
DynamicSet<OnlineUpgradeListener> listeners,
Collection<IndexDefinition<?, ?, ?>> defs,
ElasticIndexVersionDiscovery versionDiscovery) {
super(sitePaths, listeners, defs, VersionManager.getOnlineUpgrade(cfg));
this.versionDiscovery = versionDiscovery;
prefix = MoreObjects.firstNonNull(cfg.getString("index", null, "prefix"), "gerrit");
}
@Override
protected <V> boolean isDirty(Collection<Version<V>> inUse, Version<V> v) {
return !inUse.contains(v);
}
@Override
protected <K, V, I extends Index<K, V>> TreeMap<Integer, Version<V>> scanVersions(
IndexDefinition<K, V, I> def, GerritIndexStatus cfg) {
TreeMap<Integer, Version<V>> versions = new TreeMap<>();
for (Schema<V> schema : def.getSchemas().values()) {
int v = schema.getVersion();
versions.put(v, new Version<>(schema, v, cfg.getReady(def.getName(), v)));
}
try {
for (String version : versionDiscovery.discover(prefix, def.getName())) {
Integer v = Ints.tryParse(version);
if (v == null || version.length() != 4) {
log.warn("Unrecognized version in index {}: {}", def.getName(), version);
continue;
}
if (!versions.containsKey(v)) {
versions.put(v, new Version<V>(null, v, cfg.getReady(def.getName(), v)));
}
}
} catch (IOException e) {
log.error("Error scanning index: " + def.getName(), e);
}
return versions;
}
}

54
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/JestClientBuilder.java
View File

@@ -1,54 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.client.config.HttpClientConfig.Builder;
import io.searchbox.client.http.JestHttpClient;
import java.util.concurrent.TimeUnit;
@Singleton
class JestClientBuilder {
private final ElasticConfiguration cfg;
@Inject
JestClientBuilder(ElasticConfiguration cfg) {
this.cfg = cfg;
}
JestHttpClient build() {
JestClientFactory factory = new JestClientFactory();
Builder builder =
new HttpClientConfig.Builder(cfg.urls)
.multiThreaded(true)
.discoveryEnabled(false)
.connTimeout((int) cfg.connectionTimeout)
.maxConnectionIdleTime(cfg.maxConnectionIdleTime, cfg.maxConnectionIdleUnit)
.maxTotalConnection(cfg.maxTotalConnection)
.readTimeout(cfg.readTimeout)
.requestCompressionEnabled(cfg.requestCompression)
.discoveryFrequency(1L, TimeUnit.MINUTES);
if (cfg.username != null && cfg.password != null) {
builder.defaultCredentials(cfg.username, cfg.password);
}
factory.setHttpClientConfig(builder.build());
return (JestHttpClient) factory.getObject();
}
}

65
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryAccountsTest.java
View File

@@ -1,65 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryAccountsTest extends AbstractQueryAccountsTest {
private static ElasticNodeInfo nodeInfo;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
ElasticTestUtils.createAllIndexes(nodeInfo);
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
}
}
@After
public void cleanupIndex() {
if (nodeInfo != null) {
ElasticTestUtils.deleteAllIndexes(nodeInfo);
ElasticTestUtils.createAllIndexes(nodeInfo);
}
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}

77
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryChangesTest.java
View File

@@ -1,77 +0,0 @@
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.change.AbstractQueryChangesTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.gerrit.testutil.InMemoryRepositoryManager.Repo;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class ElasticQueryChangesTest extends AbstractQueryChangesTest {
private static ElasticNodeInfo nodeInfo;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
ElasticTestUtils.createAllIndexes(nodeInfo);
}
@After
public void cleanupIndex() {
if (nodeInfo != null) {
ElasticTestUtils.deleteAllIndexes(nodeInfo);
ElasticTestUtils.createAllIndexes(nodeInfo);
}
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
}
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
@Test
public void byOwnerInvalidQuery() throws Exception {
TestRepository<Repo> repo = createProject("repo");
insert(repo, newChange(repo), userId);
String nameEmail = user.asIdentifiedUser().getNameEmail();
assertQuery("owner: \"" + nameEmail + "\"\\");
}
}

65
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryGroupsTest.java
View File

@@ -1,65 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryGroupsTest extends AbstractQueryGroupsTest {
private static ElasticNodeInfo nodeInfo;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
ElasticTestUtils.createAllIndexes(nodeInfo);
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
}
}
@After
public void cleanupIndex() {
if (nodeInfo != null) {
ElasticTestUtils.deleteAllIndexes(nodeInfo);
ElasticTestUtils.createAllIndexes(nodeInfo);
}
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}

65
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java
View File

@@ -1,65 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.project.AbstractQueryProjectsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryProjectsTest extends AbstractQueryProjectsTest {
private static ElasticNodeInfo nodeInfo;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
ElasticTestUtils.createAllIndexes(nodeInfo);
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
}
}
@After
public void cleanupIndex() {
if (nodeInfo != null) {
ElasticTestUtils.deleteAllIndexes(nodeInfo);
ElasticTestUtils.createAllIndexes(nodeInfo);
}
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}

203
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
View File

@@ -1,203 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import static com.google.common.truth.Truth.assertThat;
import static com.google.gerrit.elasticsearch.ElasticAccountIndex.ACCOUNTS_PREFIX;
import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CHANGES_PREFIX;
import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CLOSED_CHANGES;
import static com.google.gerrit.elasticsearch.ElasticChangeIndex.OPEN_CHANGES;
import static com.google.gerrit.elasticsearch.ElasticGroupIndex.GROUPS_PREFIX;
import static com.google.gerrit.elasticsearch.ElasticProjectIndex.PROJECTS_PREFIX;
import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.gerrit.elasticsearch.ElasticAccountIndex.AccountMapping;
import com.google.gerrit.elasticsearch.ElasticChangeIndex.ChangeMapping;
import com.google.gerrit.elasticsearch.ElasticGroupIndex.GroupMapping;
import com.google.gerrit.elasticsearch.ElasticProjectIndex.ProjectMapping;
import com.google.gerrit.index.Schema;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.gerrit.server.index.project.ProjectSchemaDefinitions;
import com.google.gerrit.server.project.ProjectData;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.io.File;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
final class ElasticTestUtils {
static final Gson gson =
new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();
static class ElasticNodeInfo {
final Node node;
final String port;
final File elasticDir;
private ElasticNodeInfo(Node node, File rootDir, String port) {
this.node = node;
this.port = port;
this.elasticDir = rootDir;
}
}
static void configure(Config config, String port) {
config.setEnum("index", null, "type", IndexType.ELASTICSEARCH);
config.setString("elasticsearch", "test", "protocol", "http");
config.setString("elasticsearch", "test", "hostname", "localhost");
config.setString("elasticsearch", "test", "port", port);
}
static ElasticNodeInfo startElasticsearchNode() throws InterruptedException, ExecutionException {
File elasticDir = Files.createTempDir();
Path elasticDirPath = elasticDir.toPath();
Settings settings =
Settings.settingsBuilder()
.put("cluster.name", "gerrit")
.put("node.name", "Gerrit Elasticsearch Test Node")
.put("node.local", true)
.put("discovery.zen.ping.multicast.enabled", false)
.put("index.store.fs.memory.enabled", true)
.put("index.gateway.type", "none")
.put("index.max_result_window", Integer.MAX_VALUE)
.put("gateway.type", "default")
.put("http.port", 0)
.put("discovery.zen.ping.unicast.hosts", "[\"localhost\"]")
.put("path.home", elasticDirPath.toAbsolutePath())
.put("path.data", elasticDirPath.resolve("data").toAbsolutePath())
.put("path.work", elasticDirPath.resolve("work").toAbsolutePath())
.put("path.logs", elasticDirPath.resolve("logs").toAbsolutePath())
.put("transport.tcp.connect_timeout", "60s")
.build();
// Start the node
Node node = NodeBuilder.nodeBuilder().settings(settings).node();
// Wait for it to be ready
node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
assertThat(node.isClosed()).isFalse();
return new ElasticNodeInfo(node, elasticDir, getHttpPort(node));
}
static void deleteAllIndexes(ElasticNodeInfo nodeInfo) {
nodeInfo.node.client().admin().indices().prepareDelete("_all").execute();
}
static class NodeInfo {
String httpAddress;
}
static class Info {
Map<String, NodeInfo> nodes;
}
static void createAllIndexes(ElasticNodeInfo nodeInfo) {
Schema<ChangeData> changeSchema = ChangeSchemaDefinitions.INSTANCE.getLatest();
ChangeMapping openChangesMapping = new ChangeMapping(changeSchema);
ChangeMapping closedChangesMapping = new ChangeMapping(changeSchema);
openChangesMapping.closedChanges = null;
closedChangesMapping.openChanges = null;
nodeInfo
.node
.client()
.admin()
.indices()
.prepareCreate(String.format("%s%04d", CHANGES_PREFIX, changeSchema.getVersion()))
.addMapping(OPEN_CHANGES, gson.toJson(openChangesMapping))
.addMapping(CLOSED_CHANGES, gson.toJson(closedChangesMapping))
.execute()
.actionGet();
Schema<AccountState> accountSchema = AccountSchemaDefinitions.INSTANCE.getLatest();
AccountMapping accountMapping = new AccountMapping(accountSchema);
nodeInfo
.node
.client()
.admin()
.indices()
.prepareCreate(String.format("%s%04d", ACCOUNTS_PREFIX, accountSchema.getVersion()))
.addMapping(ElasticAccountIndex.ACCOUNTS, gson.toJson(accountMapping))
.execute()
.actionGet();
Schema<InternalGroup> groupSchema = GroupSchemaDefinitions.INSTANCE.getLatest();
GroupMapping groupMapping = new GroupMapping(groupSchema);
nodeInfo
.node
.client()
.admin()
.indices()
.prepareCreate(String.format("%s%04d", GROUPS_PREFIX, groupSchema.getVersion()))
.addMapping(ElasticGroupIndex.GROUPS, gson.toJson(groupMapping))
.execute()
.actionGet();
Schema<ProjectData> projectSchema = ProjectSchemaDefinitions.INSTANCE.getLatest();
ProjectMapping projectMapping = new ProjectMapping(projectSchema);
nodeInfo
.node
.client()
.admin()
.indices()
.prepareCreate(String.format("%s%04d", PROJECTS_PREFIX, projectSchema.getVersion()))
.addMapping(ElasticProjectIndex.PROJECTS, gson.toJson(projectMapping))
.execute()
.actionGet();
}
private static String getHttpPort(Node node) throws InterruptedException, ExecutionException {
String nodes =
node.client().admin().cluster().nodesInfo(new NodesInfoRequest("*")).get().toString();
Gson gson =
new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();
Info info = gson.fromJson(nodes, Info.class);
if (info.nodes == null || info.nodes.size() != 1) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
Iterator<NodeInfo> values = info.nodes.values().iterator();
String httpAddress = values.next().httpAddress;
if (Strings.isNullOrEmpty(httpAddress)) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
if (httpAddress.indexOf(':') < 0) {
throw new RuntimeException("Seems that port is not included in Elasticsearch http_address");
}
return httpAddress.substring(httpAddress.indexOf(':') + 1, httpAddress.length());
}
private ElasticTestUtils() {
// hide default constructor
}
}