Merge branch 'stable-2.14' into stable-2.15

* stable-2.14:
  ElasticReindexIT: Add tests against Elasticsearch version 6
  Elasticsearch: Add tests for queries against version 6
  Elasticsearch: Add support for V6 / one index type

The changes done in I04f275dd9 ("ElasticReindexIT: Add tests against
Elasticsearch version 6") are not included in this merge since the
reindex tests currently don't work on stable-2.15 (see issue 8799).

Change-Id: I1c940f6a95d39710dc7fb8c1cac605fa4c8efac6
This commit is contained in:
David Pursehouse
2018-06-10 13:47:19 +09:00
14 changed files with 315 additions and 38 deletions

View File

@@ -2892,10 +2892,13 @@ Sample Lucene index configuration:
=== Section elasticsearch
WARNING: The Elasticsearch support has only been tested with Elasticsearch
version 2.4.x. Support for other versions is not guaranteed.
versions 2.4, 5.6 and 6.2. Support for other versions is not guaranteed.
Open and closed changes are indexed in a single index, separated
into types `open_changes` and `closed_changes` respectively.
Open and closed changes are indexed in a single index, separated into types
`open_changes` and `closed_changes` respectively, if using Elasticsearch
versions 2.4 or 5.6. Open and closed changes are merged into the default `_doc`
type otherwise. The latter is also used for accounts and groups indices starting
with Elasticsearch 6.2.
[[elasticsearch.prefix]]elasticsearch.prefix::
+

View File

@@ -269,6 +269,15 @@ The following values are currently supported for the group name:
* server
* ssh
[[elasticsearch]]
=== Elasticsearch
Successfully running the elasticsearch tests may require setting the local
link:https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html[virtual memory].
Bazel link:https://github.com/bazelbuild/bazel/issues/3476[does not currently make container failures visible],
if any.
== Dependencies
Dependency JARs are normally downloaded as needed, but you can

View File

@@ -20,6 +20,7 @@ import static org.apache.commons.codec.binary.Base64.decodeBase64;
import com.google.common.collect.FluentIterable;
import com.google.common.io.CharStreams;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.elasticsearch.bulk.DeleteRequest;
import com.google.gerrit.index.Index;
@@ -50,6 +51,7 @@ import org.elasticsearch.client.Response;
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected static final String BULK = "_bulk";
protected static final String MAPPINGS = "mappings";
protected static final String ORDER = "order";
protected static final String SEARCH = "_search";
@@ -79,6 +81,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
private final String type;
protected final ElasticRestClientProvider client;
protected final String indexName;
@@ -98,6 +101,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
this.indexName = cfg.getIndexName(indexName, schema.getVersion());
this.indexNameRaw = indexName;
this.client = client;
this.type = client.adapter().getType(indexName);
}
@Override
@@ -117,7 +121,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void delete(K id) throws IOException {
String uri = getURI(indexNameRaw, BULK);
String uri = getURI(type, BULK);
Response response = postRequest(getDeleteActions(id), uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -156,8 +160,20 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected abstract String getId(V v);
protected String getMappingsForSingleType(String candidateType, MappingProperties properties) {
return getMappingsFor(client.adapter().getType(candidateType), properties);
}
protected String getMappingsFor(String type, MappingProperties properties) {
JsonObject mappingType = new JsonObject();
mappingType.add(type, gson.toJsonTree(properties));
JsonObject mappings = new JsonObject();
mappings.add(MAPPINGS, gson.toJsonTree(mappingType));
return gson.toJson(mappings);
}
protected String delete(String type, K id) {
return new DeleteRequest(id.toString(), indexName, type).toString();
return new DeleteRequest(id.toString(), indexName, type, client.adapter()).toString();
}
protected void addNamedElement(String name, JsonObject element, JsonArray array) {

View File

@@ -16,7 +16,6 @@ package com.google.gerrit.elasticsearch;
import static com.google.gerrit.server.index.account.AccountField.ID;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
@@ -73,6 +72,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
private final AccountMapping mapping;
private final Provider<AccountCache> accountCache;
private final Schema<AccountState> schema;
private final String type;
@AssistedInject
ElasticAccountIndex(
@@ -85,14 +85,16 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema, client.adapter());
this.schema = schema;
this.type = client.adapter().getType(ACCOUNTS);
}
@Override
public void replace(AccountState as) throws IOException {
BulkRequest bulk =
new IndexRequest(getId(as), indexName, ACCOUNTS).add(new UpdateRequest<>(schema, as));
new IndexRequest(getId(as), indexName, type, client.adapter())
.add(new UpdateRequest<>(schema, as));
String uri = getURI(ACCOUNTS, BULK);
String uri = getURI(type, BULK);
Response response = postRequest(bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -111,13 +113,12 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@Override
protected String getDeleteActions(Account.Id a) {
return delete(ACCOUNTS, a);
return delete(type, a);
}
@Override
protected String getMappings() {
ImmutableMap<String, AccountMapping> mappings = ImmutableMap.of("mappings", mapping);
return gson.toJson(mappings);
return getMappingsForSingleType(ACCOUNTS, mapping.accounts);
}
@Override
@@ -152,7 +153,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
public ResultSet<AccountState> read() throws OrmException {
try {
List<AccountState> results = Collections.emptyList();
String uri = getURI(ACCOUNTS, SEARCH);
String uri = getURI(type, SEARCH);
Response response = postRequest(search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {

View File

@@ -84,11 +84,13 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
private static final Logger log = LoggerFactory.getLogger(ElasticChangeIndex.class);
static class ChangeMapping {
MappingProperties openChanges;
MappingProperties closedChanges;
public MappingProperties changes;
public MappingProperties openChanges;
public MappingProperties closedChanges;
ChangeMapping(Schema<ChangeData> schema, ElasticQueryAdapter adapter) {
MappingProperties mapping = ElasticMapping.createMapping(schema, adapter);
this.changes = mapping;
this.openChanges = mapping;
this.closedChanges = mapping;
}
@@ -102,6 +104,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
private final Provider<ReviewDb> db;
private final ChangeData.Factory changeDataFactory;
private final Schema<ChangeData> schema;
private final String type;
@Inject
ElasticChangeIndex(
@@ -116,6 +119,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
this.changeDataFactory = changeDataFactory;
this.schema = schema;
this.mapping = new ChangeMapping(schema, client.adapter());
this.type = client.adapter().getType(CHANGES);
}
@Override
@@ -135,12 +139,15 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
throw new IOException(e);
}
ElasticQueryAdapter adapter = client.adapter();
BulkRequest bulk =
new IndexRequest(getId(cd), indexName, insertIndex)
.add(new UpdateRequest<>(schema, cd))
.add(new DeleteRequest(cd.getId().toString(), indexName, deleteIndex));
new IndexRequest(getId(cd), indexName, adapter.getType(insertIndex), adapter)
.add(new UpdateRequest<>(schema, cd));
if (!adapter.usePostV5Type()) {
bulk.add(new DeleteRequest(cd.getId().toString(), indexName, deleteIndex, adapter));
}
String uri = getURI(CHANGES, BULK);
String uri = getURI(type, BULK);
Response response = postRequest(bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -155,23 +162,36 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
throws QueryParseException {
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
List<String> indexes = Lists.newArrayListWithCapacity(2);
if (client.adapter().usePostV5Type()) {
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()
|| !Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(ElasticQueryAdapter.POST_V5_TYPE);
}
} else {
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(OPEN_CHANGES);
}
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(CLOSED_CHANGES);
}
}
return new QuerySource(indexes, p, opts);
}
@Override
protected String getDeleteActions(Id c) {
if (client.adapter().usePostV5Type()) {
return delete(ElasticQueryAdapter.POST_V5_TYPE, c);
}
return delete(OPEN_CHANGES, c) + delete(CLOSED_CHANGES, c);
}
@Override
protected String getMappings() {
return gson.toJson(ImmutableMap.of("mappings", mapping));
if (client.adapter().usePostV5Type()) {
return getMappingsFor(ElasticQueryAdapter.POST_V5_TYPE, mapping.changes);
}
return gson.toJson(ImmutableMap.of(MAPPINGS, mapping));
}
@Override

View File

@@ -14,7 +14,6 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
@@ -72,6 +71,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
private final GroupMapping mapping;
private final Provider<GroupCache> groupCache;
private final Schema<InternalGroup> schema;
private final String type;
@AssistedInject
ElasticGroupIndex(
@@ -84,14 +84,16 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema, client.adapter());
this.schema = schema;
this.type = client.adapter().getType(GROUPS);
}
@Override
public void replace(InternalGroup group) throws IOException {
BulkRequest bulk =
new IndexRequest(getId(group), indexName, GROUPS).add(new UpdateRequest<>(schema, group));
new IndexRequest(getId(group), indexName, type, client.adapter())
.add(new UpdateRequest<>(schema, group));
String uri = getURI(GROUPS, BULK);
String uri = getURI(type, BULK);
Response response = postRequest(bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -110,13 +112,12 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@Override
protected String getDeleteActions(AccountGroup.UUID g) {
return delete(GROUPS, g);
return delete(type, g);
}
@Override
protected String getMappings() {
ImmutableMap<String, GroupMapping> mappings = ImmutableMap.of("mappings", mapping);
return gson.toJson(mappings);
return getMappingsForSingleType(GROUPS, mapping.groups);
}
@Override
@@ -151,7 +152,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
public ResultSet<InternalGroup> read() throws OrmException {
try {
List<InternalGroup> results = Collections.emptyList();
String uri = getURI(GROUPS, SEARCH);
String uri = getURI(type, SEARCH);
Response response = postRequest(search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {

View File

@@ -17,7 +17,11 @@ package com.google.gerrit.elasticsearch;
import com.google.gson.JsonObject;
public class ElasticQueryAdapter {
static final String POST_V5_TYPE = "_doc";
private final boolean ignoreUnmapped;
private final boolean usePostV5Type;
private final String searchFilteringName;
private final String indicesExistParam;
private final String exactFieldType;
@@ -26,6 +30,8 @@ public class ElasticQueryAdapter {
ElasticQueryAdapter(ElasticVersion version) {
this.ignoreUnmapped = version == ElasticVersion.V2_4;
this.usePostV5Type = version == ElasticVersion.V6_2;
switch (version) {
case V5_6:
case V6_2:
@@ -52,6 +58,12 @@ public class ElasticQueryAdapter {
}
}
public void setType(JsonObject properties, String type) {
if (!usePostV5Type) {
properties.addProperty("_type", type);
}
}
public String searchFilteringName() {
return searchFilteringName;
}
@@ -71,4 +83,12 @@ public class ElasticQueryAdapter {
String indexProperty() {
return indexProperty;
}
boolean usePostV5Type() {
return usePostV5Type;
}
String getType(String preV6Type) {
return usePostV5Type() ? POST_V5_TYPE : preV6Type;
}
}

View File

@@ -14,6 +14,7 @@
package com.google.gerrit.elasticsearch.bulk;
import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
import com.google.gson.JsonObject;
abstract class ActionRequest extends BulkRequest {
@@ -22,12 +23,15 @@ abstract class ActionRequest extends BulkRequest {
private final String id;
private final String index;
private final String type;
private final ElasticQueryAdapter adapter;
protected ActionRequest(String action, String id, String index, String type) {
protected ActionRequest(
String action, String id, String index, String type, ElasticQueryAdapter adapter) {
this.action = action;
this.id = id;
this.index = index;
this.type = type;
this.adapter = adapter;
}
@Override
@@ -35,7 +39,7 @@ abstract class ActionRequest extends BulkRequest {
JsonObject properties = new JsonObject();
properties.addProperty("_id", id);
properties.addProperty("_index", index);
properties.addProperty("_type", type);
adapter.setType(properties, type);
JsonObject jsonAction = new JsonObject();
jsonAction.add(action, properties);

View File

@@ -14,9 +14,11 @@
package com.google.gerrit.elasticsearch.bulk;
import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
public class DeleteRequest extends ActionRequest {
public DeleteRequest(String id, String index, String type) {
super("delete", id, index, type);
public DeleteRequest(String id, String index, String type, ElasticQueryAdapter adapter) {
super("delete", id, index, type, adapter);
}
}

View File

@@ -14,9 +14,11 @@
package com.google.gerrit.elasticsearch.bulk;
import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
public class IndexRequest extends ActionRequest {
public IndexRequest(String id, String index, String type) {
super("index", id, index, type);
public IndexRequest(String id, String index, String type, ElasticQueryAdapter adapter) {
super("index", id, index, type, adapter);
}
}

View File

@@ -47,7 +47,7 @@ public class ElasticContainer<SELF extends ElasticContainer<SELF>> extends Gener
case V5_6:
return "elasticsearch:5.6.9-alpine";
case V6_2:
return "docker.elastic.co/elasticsearch/elasticsearch:6.2.4";
return "docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4";
}
throw new IllegalStateException("No tests for version: " + version.name());
}

View File

@@ -0,0 +1,66 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticV6QueryAccountsTest extends AbstractQueryAccountsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
container = ElasticContainer.createAndStart(ElasticVersion.V6_2);
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (container != null) {
container.stop();
}
}
private String testName() {
return testName.getMethodName().toLowerCase() + "_";
}
@Override
protected void initAfterLifecycleStart() throws Exception {
super.initAfterLifecycleStart();
ElasticTestUtils.createAllIndexes(injector);
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
String indicesPrefix = testName();
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port, indicesPrefix);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}

View File

@@ -0,0 +1,67 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.change.AbstractQueryChangesTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticV6QueryChangesTest extends AbstractQueryChangesTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
container = ElasticContainer.createAndStart(ElasticVersion.V6_2);
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (container != null) {
container.stop();
}
}
private String testName() {
return testName.getMethodName().toLowerCase() + "_";
}
@Override
protected void initAfterLifecycleStart() throws Exception {
super.initAfterLifecycleStart();
ElasticTestUtils.createAllIndexes(injector);
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
String indicesPrefix = testName();
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port, indicesPrefix);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}

View File

@@ -0,0 +1,66 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticV6QueryGroupsTest extends AbstractQueryGroupsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
container = ElasticContainer.createAndStart(ElasticVersion.V6_2);
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (container != null) {
container.stop();
}
}
private String testName() {
return testName.getMethodName().toLowerCase() + "_";
}
@Override
protected void initAfterLifecycleStart() throws Exception {
super.initAfterLifecycleStart();
ElasticTestUtils.createAllIndexes(injector);
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
String indicesPrefix = testName();
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port, indicesPrefix);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}