opendev/gerrit
Code Issues Proposed changes

Elasticsearch: Discontinue EOL version 6.8 thus whole V6 support

Browse Source 
- Remove the whole V6 support as 6.8 was the last supported 6.x version.
- Remove the no longer necessary documentation parts about that version.
- Update the related tests accordingly, including removing the V6 ones.

[1] https://www.elastic.co/support/eol

Change-Id: I4748c9718eec246479a2fae78e5f9b1ef912fb93
This commit is contained in:
Marco Miller
2020-12-06 16:10:20 -05:00
parent 9134e7917e
commit 9caa81a31b
20 changed files with 31 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
View File

@@ -127,7 +127,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private final SitePaths sitePaths;
private final String indexNameRaw;
protected final String type;
protected final ElasticRestClientProvider client;
protected final String indexName;
protected final Gson gson;
@@ -147,7 +146,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
this.indexName = config.getIndexName(indexName, schema.getVersion());
this.indexNameRaw = indexName;
this.client = client;
this.type = client.adapter().getType();
}
@Override
@@ -167,7 +165,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void delete(K id) {
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, getDeleteActions(id), getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -192,10 +190,8 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
}
// Recreate the index.
String indexCreationFields = concatJsonString(getSettings(client.adapter()), getMappings());
response =
performRequest(
"PUT", indexName + client.adapter().includeTypeNameParam(), indexCreationFields);
String indexCreationFields = concatJsonString(getSettings(), getMappings());
response = performRequest("PUT", indexName, indexCreationFields);
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
String error = String.format("Failed to create index %s: %s", indexName, statusCode);
@@ -207,26 +203,20 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected abstract String getMappings();
private String getSettings(ElasticQueryAdapter adapter) {
return gson.toJson(ImmutableMap.of(SETTINGS, ElasticSetting.createSetting(config, adapter)));
private String getSettings() {
return gson.toJson(ImmutableMap.of(SETTINGS, ElasticSetting.createSetting(config)));
}
protected abstract String getId(V v);
protected String getMappingsForSingleType(MappingProperties properties) {
return getMappingsFor(client.adapter().getType(), properties);
return getMappingsFor(properties);
}
protected String getMappingsFor(String type, MappingProperties properties) {
protected String getMappingsFor(MappingProperties properties) {
JsonObject mappings = new JsonObject();
if (client.adapter().omitType()) {
mappings.add(MAPPINGS, gson.toJsonTree(properties));
} else {
JsonObject mappingType = new JsonObject();
mappingType.add(type, gson.toJsonTree(properties));
mappings.add(MAPPINGS, gson.toJsonTree(mappingType));
}
mappings.add(MAPPINGS, gson.toJsonTree(properties));
return gson.toJson(mappings);
}
@@ -305,15 +295,9 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
return sortArray;
}
protected String getURI(String type, String request) {
protected String getURI(String request) {
try {
String encodedIndexName = URLEncoder.encode(indexName, UTF_8.toString());
if (SEARCH.equals(request) && client.adapter().omitType()) {
return encodedIndexName + "/" + request;
}
String encodedTypeIfAny =
client.adapter().omitType() ? "" : "/" + URLEncoder.encode(type, UTF_8.toString());
return encodedIndexName + encodedTypeIfAny + "/" + request;
return URLEncoder.encode(indexName, UTF_8.toString()) + "/" + request;
} catch (UnsupportedEncodingException e) {
throw new StorageException(e);
}
@@ -359,12 +343,10 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected class ElasticQuerySource implements DataSource<V> {
private final QueryOptions opts;
private final String search;
private final String index;
ElasticQuerySource(Predicate<V> p, QueryOptions opts, String index, JsonArray sortArray)
ElasticQuerySource(Predicate<V> p, QueryOptions opts, JsonArray sortArray)
throws QueryParseException {
this.opts = opts;
this.index = index;
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
SearchSourceBuilder searchSource =
new SearchSourceBuilder(client.adapter())
@@ -392,7 +374,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) {
try {
String uri = getURI(index, SEARCH);
String uri = getURI(SEARCH);
Response response =
performRequest(HttpPost.METHOD_NAME, uri, search, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();

3
java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
View File

@@ -74,7 +74,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
public void replace(AccountState as) {
BulkRequest bulk = new IndexRequest(getId(as), indexName).add(new UpdateRequest<>(schema, as));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -96,7 +96,6 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
return new ElasticQuerySource(
p,
opts.filterFields(o -> IndexUtils.accountFields(o, schema.useLegacyNumericFields())),
type,
sortArray);
}

35
java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
View File

@@ -14,8 +14,6 @@
package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.CLOSED_STATUSES;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.OPEN_STATUSES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
@@ -23,7 +21,6 @@ import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
@@ -50,7 +47,6 @@ import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.project.SubmitRuleOptions;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gson.JsonArray;
@@ -59,7 +55,6 @@ import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.http.HttpStatus;
@@ -82,8 +77,6 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
}
private static final String CHANGES = "changes";
private static final String OPEN_CHANGES = "open_" + CHANGES;
private static final String CLOSED_CHANGES = "closed_" + CHANGES;
private final ChangeMapping mapping;
private final ChangeData.Factory changeDataFactory;
@@ -109,7 +102,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
public void replace(ChangeData cd) {
BulkRequest bulk = new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -122,27 +115,9 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
@Override
public DataSource<ChangeData> getSource(Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
List<String> indexes = Lists.newArrayListWithCapacity(2);
if (!client.adapter().omitType()) {
if (client.adapter().useV6Type()) {
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()
|| !Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(ElasticQueryAdapter.V6_TYPE);
}
} else {
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(OPEN_CHANGES);
}
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(CLOSED_CHANGES);
}
}
}
QueryOptions filteredOpts =
opts.filterFields(o -> IndexUtils.changeFields(o, schema.useLegacyNumericFields()));
return new ElasticQuerySource(p, filteredOpts, getURI(indexes), getSortArray());
return new ElasticQuerySource(p, filteredOpts, getSortArray());
}
private JsonArray getSortArray() {
@@ -155,10 +130,6 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
return sortArray;
}
private String getURI(List<String> types) {
return String.join(",", types);
}
@Override
protected String getDeleteActions(Change.Id c) {
return getDeleteRequest(c);
@@ -166,7 +137,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
@Override
protected String getMappings() {
return getMappingsFor(client.adapter().getType(), mapping.changes);
return getMappingsFor(mapping.changes);
}
@Override

7
java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
View File

@@ -44,7 +44,7 @@ class ElasticConfiguration {
static final String DEFAULT_PORT = "9200";
static final String DEFAULT_USERNAME = "elastic";
static final int DEFAULT_NUMBER_OF_SHARDS = 0;
static final int DEFAULT_NUMBER_OF_SHARDS = 1;
static final int DEFAULT_NUMBER_OF_REPLICAS = 1;
static final int DEFAULT_MAX_RESULT_WINDOW = 10000;
@@ -107,10 +107,7 @@ class ElasticConfiguration {
return String.format("%s%s_%04d", prefix, name, schemaVersion);
}
int getNumberOfShards(ElasticQueryAdapter adapter) {
if (numberOfShards == DEFAULT_NUMBER_OF_SHARDS) {
return adapter.getDefaultNumberOfShards();
}
int getNumberOfShards() {
return numberOfShards;
}
}

4
java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
View File

@@ -75,7 +75,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
BulkRequest bulk =
new IndexRequest(getId(group), indexName).add(new UpdateRequest<>(schema, group));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -90,7 +90,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
JsonArray sortArray = getSortArray(GroupField.UUID.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), type, sortArray);
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), sortArray);
}
@Override

4
java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
View File

@@ -76,7 +76,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
new IndexRequest(projectState.getProject().getName(), indexName)
.add(new UpdateRequest<>(schema, projectState));
String uri = getURI(type, BULK);
String uri = getURI(BULK);
Response response = postRequest(uri, bulk, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -91,7 +91,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
JsonArray sortArray = getSortArray(ProjectField.NAME.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), type, sortArray);
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), sortArray);
}
@Override

45
java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java
View File

@@ -14,42 +14,23 @@
package com.google.gerrit.elasticsearch;
import static com.google.gerrit.elasticsearch.ElasticVersion.V6_8;
public class ElasticQueryAdapter {
static final String V6_TYPE = "_doc";
private static final String INCLUDE_TYPE = "include_type_name=true";
private static final String INDICES = "?allow_no_indices=false";
private final boolean useV6Type;
private final boolean omitType;
private final int defaultNumberOfShards;
private final String searchFilteringName;
private final String indicesExistParams;
private final String exactFieldType;
private final String stringFieldType;
private final String indexProperty;
private final String rawFieldsKey;
private final String versionDiscoveryUrl;
private final String includeTypeNameParam;
ElasticQueryAdapter(ElasticVersion version) {
this.useV6Type = version.isV6();
this.omitType = version.isV7OrLater();
this.defaultNumberOfShards = version.isV7OrLater() ? 1 : 5;
this.versionDiscoveryUrl = version.isV6OrLater() ? "/%s*" : "/%s*/_aliases";
ElasticQueryAdapter() {
this.versionDiscoveryUrl = "/%s*";
this.searchFilteringName = "_source";
this.exactFieldType = "keyword";
this.stringFieldType = "text";
this.indexProperty = "true";
this.rawFieldsKey = "_source";
// Since v6.7 (end-of-life), in fact, for these two parameters:
this.indicesExistParams =
version.isAtLeastMinorVersion(V6_8) ? INDICES + "&" + INCLUDE_TYPE : INDICES;
this.includeTypeNameParam = version.isAtLeastMinorVersion(V6_8) ? "?" + INCLUDE_TYPE : "";
}
public String searchFilteringName() {
@@ -57,7 +38,7 @@ public class ElasticQueryAdapter {
}
String indicesExistParams() {
return indicesExistParams;
return INDICES;
}
String exactFieldType() {
@@ -76,27 +57,7 @@ public class ElasticQueryAdapter {
return rawFieldsKey;
}
boolean useV6Type() {
return useV6Type;
}
boolean omitType() {
return omitType;
}
int getDefaultNumberOfShards() {
return defaultNumberOfShards;
}
String getType() {
return useV6Type() ? V6_TYPE : "";
}
String getVersionDiscoveryUrl(String name) {
return String.format(versionDiscoveryUrl, name);
}
String includeTypeNameParam() {
return includeTypeNameParam;
}
}

2
java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java
View File

@@ -65,7 +65,7 @@ class ElasticRestClientProvider implements Provider<RestClient>, LifecycleListen
client = build();
ElasticVersion version = getVersion();
logger.atInfo().log("Elasticsearch integration version %s", version);
adapter = new ElasticQueryAdapter(version);
adapter = new ElasticQueryAdapter();
}
}
}

8
java/com/google/gerrit/elasticsearch/ElasticSetting.java
View File

@@ -22,18 +22,18 @@ class ElasticSetting {
private static final ImmutableMap<String, String> CUSTOM_CHAR_MAPPING =
ImmutableMap.of("\\u002E", "\\u0020", "\\u005F", "\\u0020");
static SettingProperties createSetting(ElasticConfiguration config, ElasticQueryAdapter adapter) {
return new ElasticSetting.Builder().addCharFilter().addAnalyzer().build(config, adapter);
static SettingProperties createSetting(ElasticConfiguration config) {
return new ElasticSetting.Builder().addCharFilter().addAnalyzer().build(config);
}
static class Builder {
private final ImmutableMap.Builder<String, FieldProperties> fields =
new ImmutableMap.Builder<>();
SettingProperties build(ElasticConfiguration config, ElasticQueryAdapter adapter) {
SettingProperties build(ElasticConfiguration config) {
SettingProperties properties = new SettingProperties();
properties.analysis = fields.build();
properties.numberOfShards = config.getNumberOfShards(adapter);
properties.numberOfShards = config.getNumberOfShards();
properties.numberOfReplicas = config.numberOfReplicas;
properties.maxResultWindow = config.maxResultWindow;
return properties;

29
java/com/google/gerrit/elasticsearch/ElasticVersion.java
View File

@@ -18,7 +18,6 @@ import com.google.common.base.Joiner;
import java.util.regex.Pattern;
public enum ElasticVersion {
V6_8("6.8.*"),
V7_0("7.0.*"),
V7_1("7.1.*"),
V7_2("7.2.*"),
@@ -67,34 +66,6 @@ public enum ElasticVersion {
return Joiner.on(", ").join(ElasticVersion.values());
}
public boolean isV6() {
return getMajor() == 6;
}
public boolean isV6OrLater() {
return isAtLeastVersion(6);
}
public boolean isV7OrLater() {
return isAtLeastVersion(7);
}
private boolean isAtLeastVersion(int major) {
return getMajor() >= major;
}
public boolean isAtLeastMinorVersion(ElasticVersion version) {
return getMajor().equals(version.getMajor()) && getMinor() >= version.getMinor();
}
private Integer getMajor() {
return Integer.valueOf(version.split("\\.")[0]);
}
private Integer getMinor() {
return Integer.valueOf(version.split("\\.")[1]);
}
@Override
public String toString() {
return version;