opendev/gerrit
Code Issues Proposed changes

Merge branch 'stable-2.14' into stable-2.15

Browse Source 
* stable-2.14:
  ElasticContainer: Allow to specify the docker container version to create
  ElasticContainer: Include cause in AssumptionViolatedException
  ElasticContainer: Create with static method
  Acceptance tests: Replace embedded ES with docker testcontainer
  Elasticsearch: replace native API in prod w/ REST

Changes to ReindexIT done in Iccf443102 ("Acceptance tests: Replace embedded
ES with docker testcontainer") are reverted in this merge since that test
currently doesn't work with Elasticsearch on stable-2.15 (issue 8799).

Change-Id: I21d8b10ebd450c7dc840846a5a0d836b4243dacc
This commit is contained in:
David Pursehouse
2018-05-28 19:24:39 +09:00
parent e65498896f c337290165
commit d92ba3bc8d
37 changed files with 1451 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java Normal file
View File

@@ -0,0 +1,81 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.apache.http.HttpHost;
import org.junit.internal.AssumptionViolatedException;
import org.testcontainers.containers.GenericContainer;
/* Helper class for running ES integration tests in docker container */
public class ElasticContainer<SELF extends ElasticContainer<SELF>> extends GenericContainer<SELF> {
private static final int ELASTICSEARCH_DEFAULT_PORT = 9200;
public enum Version {
V2,
V5,
V6
}
public static ElasticContainer<?> createAndStart(Version version) {
// Assumption violation is not natively supported by Testcontainers.
// See https://github.com/testcontainers/testcontainers-java/issues/343
try {
ElasticContainer<?> container = new ElasticContainer<>(version);
container.start();
return container;
} catch (Throwable t) {
throw new AssumptionViolatedException("Unable to start container", t);
}
}
public static ElasticContainer<?> createAndStart() {
return createAndStart(Version.V2);
}
private static String getImageName(Version version) {
switch (version) {
case V2:
return "elasticsearch:2.4.6-alpine";
case V5:
return "elasticsearch:5.6.9-alpine";
case V6:
return "docker.elastic.co/elasticsearch/elasticsearch:6.2.4";
}
throw new IllegalStateException("Unsupported version: " + version.name());
}
private ElasticContainer(Version version) {
super(getImageName(version));
}
@Override
protected void configure() {
addExposedPort(ELASTICSEARCH_DEFAULT_PORT);
// https://github.com/docker-library/elasticsearch/issues/58
addEnv("-Ees.network.host", "0.0.0.0");
}
@Override
protected Set<Integer> getLivenessCheckPorts() {
return ImmutableSet.of(getMappedPort(ELASTICSEARCH_DEFAULT_PORT));
}
public HttpHost getHttpHost() {
return new HttpHost(getContainerIpAddress(), getMappedPort(ELASTICSEARCH_DEFAULT_PORT));
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryAccountsTest.java
View File

@@ -19,29 +19,29 @@ import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryAccountsTest extends AbstractQueryAccountsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryChangesTest.java
View File

@@ -20,7 +20,6 @@ import com.google.gerrit.testutil.InMemoryModule;
import com.google.gerrit.testutil.InMemoryRepositoryManager.Repo;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
@@ -29,22 +28,23 @@ import org.junit.Test;
public class ElasticQueryChangesTest extends AbstractQueryChangesTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

14
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryGroupsTest.java
View File

@@ -19,29 +19,29 @@ import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryGroupsTest extends AbstractQueryGroupsTest {
private static ElasticNodeInfo nodeInfo;
private static ElasticContainer<?> container;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
public static void startIndexService() {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
container = ElasticContainer.createAndStart();
nodeInfo = new ElasticNodeInfo(container.getHttpHost().getPort());
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
if (container != null) {
container.stop();
}
}

91
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
View File

@@ -14,90 +14,31 @@
package com.google.gerrit.elasticsearch;
import static com.google.common.truth.Truth.assertThat;
import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
public final class ElasticTestUtils {
public static class ElasticNodeInfo {
public final Node node;
public final String port;
public final File elasticDir;
public final int port;
private ElasticNodeInfo(Node node, File rootDir, String port) {
this.node = node;
public ElasticNodeInfo(int port) {
this.port = port;
this.elasticDir = rootDir;
}
}
static void configure(Config config, String port, String prefix) {
public static void configure(Config config, int port, String prefix) {
config.setEnum("index", null, "type", IndexType.ELASTICSEARCH);
config.setString("elasticsearch", "test", "protocol", "http");
config.setString("elasticsearch", "test", "hostname", "localhost");
config.setString("elasticsearch", "test", "port", port);
config.setInt("elasticsearch", "test", "port", port);
config.setString("elasticsearch", null, "prefix", prefix);
}
static ElasticNodeInfo startElasticsearchNode() throws InterruptedException, ExecutionException {
File elasticDir = Files.createTempDir();
Path elasticDirPath = elasticDir.toPath();
Settings settings =
Settings.settingsBuilder()
.put("cluster.name", "gerrit")
.put("node.name", "Gerrit Elasticsearch Test Node")
.put("node.local", true)
.put("discovery.zen.ping.multicast.enabled", false)
.put("index.store.fs.memory.enabled", true)
.put("index.gateway.type", "none")
.put("index.max_result_window", Integer.MAX_VALUE)
.put("gateway.type", "default")
.put("http.port", 0)
.put("discovery.zen.ping.unicast.hosts", "[\"localhost\"]")
.put("path.home", elasticDirPath.toAbsolutePath())
.put("path.data", elasticDirPath.resolve("data").toAbsolutePath())
.put("path.work", elasticDirPath.resolve("work").toAbsolutePath())
.put("path.logs", elasticDirPath.resolve("logs").toAbsolutePath())
.put("transport.tcp.connect_timeout", "60s")
.build();
// Start the node
Node node = NodeBuilder.nodeBuilder().settings(settings).node();
// Wait for it to be ready
node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
assertThat(node.isClosed()).isFalse();
return new ElasticNodeInfo(node, elasticDir, getHttpPort(node));
}
static class NodeInfo {
String httpAddress;
}
static class Info {
Map<String, NodeInfo> nodes;
config.setInt("index", null, "maxLimit", 10000);
}
public static void createAllIndexes(Injector injector) throws IOException {
@@ -108,28 +49,6 @@ public final class ElasticTestUtils {
}
}
private static String getHttpPort(Node node) throws InterruptedException, ExecutionException {
String nodes =
node.client().admin().cluster().nodesInfo(new NodesInfoRequest("*")).get().toString();
Gson gson =
new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();
Info info = gson.fromJson(nodes, Info.class);
if (info.nodes == null || info.nodes.size() != 1) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
Iterator<NodeInfo> values = info.nodes.values().iterator();
String httpAddress = values.next().httpAddress;
if (Strings.isNullOrEmpty(httpAddress)) {
throw new RuntimeException("Cannot extract local Elasticsearch http port");
}
if (httpAddress.indexOf(':') < 0) {
throw new RuntimeException("Seems that port is not included in Elasticsearch http_address");
}
return httpAddress.substring(httpAddress.indexOf(':') + 1, httpAddress.length());
}
private ElasticTestUtils() {
// hide default constructor
}