opendev/gerrit
Code Issues Proposed changes

Merge branch 'stable-3.2' into stable-3.3

Browse Source 
* stable-3.2:
  Update git submodules
  Update git submodules
  Update git submodules
  Compact the REST-API output JSON unconditionally
  Elasticsearch: Discontinue EOL versions 7.0 and 7.1 support
  Elasticsearch: Discontinue EOL version 6.8 thus whole V6 support
  Update git submodules
  Avoid logging "length=0" exception

Change-Id: I4cae34338b02fa7fa8c016e0158be261a77b87c6
This commit is contained in:
Marco Miller
2020-12-15 14:05:21 -05:00
parent 8114715235 fa79dafbee
commit 062603e89b
21 changed files with 47 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@@ -127,7 +127,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private final SitePaths sitePaths;
private final String indexNameRaw;
protected final String type;
protected final ElasticRestClientProvider client;
protected final String indexName;
protected final Gson gson;
@@ -147,7 +146,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
this.indexName = config.getIndexName(indexName, schema.getVersion());
this.indexNameRaw = indexName;
this.client = client;
this.type = client.adapter().getType();
}
@Override
@@ -167,7 +165,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void delete(K id) {
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, getDeleteActions(id), getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -192,10 +190,8 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
// Recreate the index.
String indexCreationFields = concatJsonString(getSettings(client.adapter()), getMappings());
response =
performRequest(
"PUT", indexName + client.adapter().includeTypeNameParam(), indexCreationFields);
String indexCreationFields = concatJsonString(getSettings(), getMappings());
response = performRequest("PUT", indexName, indexCreationFields);
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
String error = String.format("Failed to create index %s: %s", indexName, statusCode);
@@ -207,26 +203,20 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected abstract String getMappings();
private String getSettings(ElasticQueryAdapter adapter) {
return gson.toJson(ImmutableMap.of(SETTINGS, ElasticSetting.createSetting(config, adapter)));
private String getSettings() {
return gson.toJson(ImmutableMap.of(SETTINGS, ElasticSetting.createSetting(config)));
}
protected abstract String getId(V v);
protected String getMappingsForSingleType(MappingProperties properties) {
return getMappingsFor(client.adapter().getType(), properties);
return getMappingsFor(properties);
}
protected String getMappingsFor(String type, MappingProperties properties) {
protected String getMappingsFor(MappingProperties properties) {
JsonObject mappings = new JsonObject();
if (client.adapter().omitType()) {
mappings.add(MAPPINGS, gson.toJsonTree(properties));
} else {
JsonObject mappingType = new JsonObject();
mappingType.add(type, gson.toJsonTree(properties));
mappings.add(MAPPINGS, gson.toJsonTree(mappingType));
}
mappings.add(MAPPINGS, gson.toJsonTree(properties));
return gson.toJson(mappings);
}
@@ -305,15 +295,9 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
return sortArray;
}
protected String getURI(String type, String request) {
protected String getURI(String request) {
try {
String encodedIndexName = URLEncoder.encode(indexName, UTF_8.toString());
if (SEARCH.equals(request) && client.adapter().omitType()) {
return encodedIndexName + "/" + request;
}
String encodedTypeIfAny =
client.adapter().omitType() ? "" : "/" + URLEncoder.encode(type, UTF_8.toString());
return encodedIndexName + encodedTypeIfAny + "/" + request;
return URLEncoder.encode(indexName, UTF_8.toString()) + "/" + request;
} catch (UnsupportedEncodingException e) {
throw new StorageException(e);
}
@@ -359,12 +343,10 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected class ElasticQuerySource implements DataSource<V> {
private final QueryOptions opts;
private final String search;
private final String index;
ElasticQuerySource(Predicate<V> p, QueryOptions opts, String index, JsonArray sortArray)
ElasticQuerySource(Predicate<V> p, QueryOptions opts, JsonArray sortArray)
throws QueryParseException {
this.opts = opts;
this.index = index;
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
SearchSourceBuilder searchSource =
new SearchSourceBuilder(client.adapter())
@@ -392,7 +374,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) {
try {
String uri = getURI(index, SEARCH);
String uri = getURI(SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, uri, search, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();

3
java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@@ -77,7 +77,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
new IndexRequest(getId(as), indexName)
.add(new UpdateRequest<>(schema, as, ImmutableSet.of()));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -99,7 +99,6 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
return new ElasticQuerySource(
p,
opts.filterFields(o -> IndexUtils.accountFields(o, schema.useLegacyNumericFields())),
type,
sortArray);
}

35
java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@@ -14,8 +14,6 @@
package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.CLOSED_STATUSES;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.OPEN_STATUSES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
@@ -24,7 +22,6 @@ import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
@@ -53,7 +50,6 @@ import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.project.SubmitRuleOptions;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gson.JsonArray;
@@ -62,7 +58,6 @@ import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.http.HttpStatus;
@@ -86,8 +81,6 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
private static final String CHANGES = "changes";
private static final String OPEN_CHANGES = "open_" + CHANGES;
private static final String CLOSED_CHANGES = "closed_" + CHANGES;
private final ChangeMapping mapping;
private final ChangeData.Factory changeDataFactory;
@@ -120,7 +113,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
BulkRequest bulk =
new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd, skipFields));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -133,27 +126,9 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
@Override
public DataSource<ChangeData> getSource(Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
List<String> indexes = Lists.newArrayListWithCapacity(2);
if (!client.adapter().omitType()) {
if (client.adapter().useV6Type()) {
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()
|| !Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(ElasticQueryAdapter.V6_TYPE);
}
} else {
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(OPEN_CHANGES);
}
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(CLOSED_CHANGES);
}
}
}
QueryOptions filteredOpts =
opts.filterFields(o -> IndexUtils.changeFields(o, schema.useLegacyNumericFields()));
return new ElasticQuerySource(p, filteredOpts, getURI(indexes), getSortArray());
return new ElasticQuerySource(p, filteredOpts, getSortArray());
}
private JsonArray getSortArray() {
@@ -166,10 +141,6 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
return sortArray;
}
private String getURI(List<String> types) {
return String.join(",", types);
}
@Override
protected String getDeleteActions(Change.Id c) {
return getDeleteRequest(c);
@@ -177,7 +148,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
@Override
protected String getMappings() {
return getMappingsFor(client.adapter().getType(), mapping.changes);
return getMappingsFor(mapping.changes);
}
@Override

7
java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
View File

@@ -44,7 +44,7 @@ class ElasticConfiguration {
static final String DEFAULT_PORT = "9200";
static final String DEFAULT_USERNAME = "elastic";
static final int DEFAULT_NUMBER_OF_SHARDS = 0;
static final int DEFAULT_NUMBER_OF_SHARDS = 1;
static final int DEFAULT_NUMBER_OF_REPLICAS = 1;
static final int DEFAULT_MAX_RESULT_WINDOW = 10000;
@@ -107,10 +107,7 @@ class ElasticConfiguration {
return String.format("%s%s_%04d", prefix, name, schemaVersion);
}
int getNumberOfShards(ElasticQueryAdapter adapter) {
if (numberOfShards == DEFAULT_NUMBER_OF_SHARDS) {
return adapter.getDefaultNumberOfShards();
}
int getNumberOfShards() {
return numberOfShards;
}
}

4
java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@@ -77,7 +77,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
new IndexRequest(getId(group), indexName)
.add(new UpdateRequest<>(schema, group, ImmutableSet.of()));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -92,7 +92,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
JsonArray sortArray = getSortArray(GroupField.UUID.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), type, sortArray);
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), sortArray);
}
@Override

4
java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
View File

@@ -79,7 +79,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
new IndexRequest(projectState.getProject().getName(), indexName)
.add(new UpdateRequest<>(schema, projectState, ImmutableSet.of()));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -94,7 +94,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
JsonArray sortArray = getSortArray(ProjectField.NAME.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), type, sortArray);
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), sortArray);
}
@Override

45
java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java
View File

@@ -14,42 +14,23 @@
package com.google.gerrit.elasticsearch;
import static com.google.gerrit.elasticsearch.ElasticVersion.V6_8;
public class ElasticQueryAdapter {
static final String V6_TYPE = "_doc";
private static final String INCLUDE_TYPE = "include_type_name=true";
private static final String INDICES = "?allow_no_indices=false";
private final boolean useV6Type;
private final boolean omitType;
private final int defaultNumberOfShards;
private final String searchFilteringName;
private final String indicesExistParams;
private final String exactFieldType;
private final String stringFieldType;
private final String indexProperty;
private final String rawFieldsKey;
private final String versionDiscoveryUrl;
private final String includeTypeNameParam;
ElasticQueryAdapter(ElasticVersion version) {
this.useV6Type = version.isV6();
this.omitType = version.isV7OrLater();
this.defaultNumberOfShards = version.isV7OrLater() ? 1 : 5;
this.versionDiscoveryUrl = version.isV6OrLater() ? "/%s*" : "/%s*/_aliases";
ElasticQueryAdapter() {
this.versionDiscoveryUrl = "/%s*";
this.searchFilteringName = "_source";
this.exactFieldType = "keyword";
this.stringFieldType = "text";
this.indexProperty = "true";
this.rawFieldsKey = "_source";
// Since v6.7 (end-of-life), in fact, for these two parameters:
this.indicesExistParams =
version.isAtLeastMinorVersion(V6_8) ? INDICES + "&" + INCLUDE_TYPE : INDICES;
this.includeTypeNameParam = version.isAtLeastMinorVersion(V6_8) ? "?" + INCLUDE_TYPE : "";
}
public String searchFilteringName() {
@@ -57,7 +38,7 @@ public class ElasticQueryAdapter {
}
String indicesExistParams() {
return indicesExistParams;
return INDICES;
}
String exactFieldType() {
@@ -76,27 +57,7 @@ public class ElasticQueryAdapter {
return rawFieldsKey;
}
boolean useV6Type() {
return useV6Type;
}
boolean omitType() {
return omitType;
}
int getDefaultNumberOfShards() {
return defaultNumberOfShards;
}
String getType() {
return useV6Type() ? V6_TYPE : "";
}
String getVersionDiscoveryUrl(String name) {
return String.format(versionDiscoveryUrl, name);
}
String includeTypeNameParam() {
return includeTypeNameParam;
}
}

2
java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java
View File

@@ -65,7 +65,7 @@ class ElasticRestClientProvider implements Provider<RestClient>, LifecycleListen
client = build();
ElasticVersion version = getVersion();
logger.atInfo().log("Elasticsearch integration version %s", version);
adapter = new ElasticQueryAdapter(version);
adapter = new ElasticQueryAdapter();
}
}
}

8
java/com/google/gerrit/elasticsearch/ElasticSetting.java
View File

@@ -22,18 +22,18 @@ class ElasticSetting {
private static final ImmutableMap<String, String> CUSTOM_CHAR_MAPPING =
ImmutableMap.of("\\u002E", "\\u0020", "\\u005F", "\\u0020");
static SettingProperties createSetting(ElasticConfiguration config, ElasticQueryAdapter adapter) {
return new ElasticSetting.Builder().addCharFilter().addAnalyzer().build(config, adapter);
static SettingProperties createSetting(ElasticConfiguration config) {
return new ElasticSetting.Builder().addCharFilter().addAnalyzer().build(config);
}
static class Builder {
private final ImmutableMap.Builder<String, FieldProperties> fields =
new ImmutableMap.Builder<>();
SettingProperties build(ElasticConfiguration config, ElasticQueryAdapter adapter) {
SettingProperties build(ElasticConfiguration config) {
SettingProperties properties = new SettingProperties();
properties.analysis = fields.build();
properties.numberOfShards = config.getNumberOfShards(adapter);
properties.numberOfShards = config.getNumberOfShards();
properties.numberOfReplicas = config.numberOfReplicas;
properties.maxResultWindow = config.maxResultWindow;
return properties;

31
java/com/google/gerrit/elasticsearch/ElasticVersion.java
View File

@@ -18,9 +18,6 @@ import com.google.common.base.Joiner;
import java.util.regex.Pattern;
public enum ElasticVersion {
V6_8("6.8.*"),
V7_0("7.0.*"),
V7_1("7.1.*"),
V7_2("7.2.*"),
V7_3("7.3.*"),
V7_4("7.4.*"),
@@ -67,34 +64,6 @@ public enum ElasticVersion {
return Joiner.on(", ").join(ElasticVersion.values());
}
public boolean isV6() {
return getMajor() == 6;
}
public boolean isV6OrLater() {
return isAtLeastVersion(6);
}
public boolean isV7OrLater() {
return isAtLeastVersion(7);
}
private boolean isAtLeastVersion(int major) {
return getMajor() >= major;
}
public boolean isAtLeastMinorVersion(ElasticVersion version) {
return getMajor().equals(version.getMajor()) && getMinor() >= version.getMinor();
}
private Integer getMajor() {
return Integer.valueOf(version.split("\\.")[0]);
}
private Integer getMinor() {
return Integer.valueOf(version.split("\\.")[1]);
}
@Override
public String toString() {
return version;

17
java/com/google/gerrit/sshd/BaseCommand.java
View File

@@ -51,6 +51,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicReference;
@@ -356,7 +357,7 @@ public abstract class BaseCommand implements Command {
}
m.append(" during ");
m.append(context.getCommandLine());
logger.atSevere().withCause(e).log(m.toString());
logCauseIfRelevant(e, m);
}
if (e instanceof Failure) {
@@ -383,6 +384,20 @@ public abstract class BaseCommand implements Command {
return 128;
}
private void logCauseIfRelevant(Throwable e, StringBuilder message) {
String zeroLength = "length=0";
String streamAlreadyClosed = "stream is already closed";
boolean isZeroLength = false;
if (streamAlreadyClosed.equals(e.getMessage())) {
StackTraceElement[] stackTrace = e.getStackTrace();
isZeroLength = Arrays.stream(stackTrace).anyMatch(s -> s.toString().contains(zeroLength));
}
if (!isZeroLength) {
logger.atSevere().withCause(e).log(message.toString());
}
}
protected UnloggedFailure die(String msg) {
return new UnloggedFailure(1, "fatal: " + msg);
}