Merge "ES: Allow to configure multiple ElasticSearch servers" into stable-2.14

Documentation/config-gerrit.txt

@@ -2687,25 +2687,9 @@ Open and closed changes are indexed in a single index, separated
 into types 'open_changes' and 'closed_changes' respectively.
 
 The following settings are only used when the index type is
-`ELASTICSEARCH`.
+`ELASTICSEARCH`. To configure Elasticsearch servers look into
+link:#elasticsearch[Elasticsearch section]
 
-[[index.protocol]]index.protocol::
-+
-Elasticsearch server protocol [http|https].
-+
-Defaults to `http`.
-
-[[index.hostname]]index.hostname::
-+
-Elasticsearch server hostname.
-
-Defaults to `localhost`.
-
-[[index.port]]index.port::
-+
-Elasticsearch server port.
-+
-Defaults to `9200`.
 
 [[index.prefix]]index.prefix::
 +
@@ -2715,6 +2699,33 @@ change index named 'gerrit1_changes_0001'.
 +
 Not set by default.
 
+[[elasticsearch]]
+=== Section elasticsearch
+
+WARNING: The Elasticsearch support is incomplete. Online reindexing
+is not implemented yet.
+
+Each section correspond to the one Elasticsearch server.
+
+[[elasticsearch.name.protocol]]elasticsearch.name.protocol::
++
+Elasticsearch server protocol [http|https].
++
+Defaults to `http`.
+
+[[elasticsearch.name.hostname]]elasticsearch.name.hostname::
++
+Elasticsearch server hostname.
+
+Defaults to `localhost`.
+
+[[elasticsearch.name.port]]elasticsearch.name.port::
++
+Elasticsearch server port.
++
+Defaults to `9200`.
+
+
 [[ldap]]
 === Section ldap
 

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java

@@ -63,7 +63,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
   private final FillArgs fillArgs;
   private final SitePaths sitePaths;
 
-  protected final boolean refresh;
   protected final String indexName;
   protected final JestHttpClient client;
   protected final Gson gson;
@@ -87,7 +86,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
             Strings.nullToEmpty(cfg.getString("index", null, "prefix")),
             indexName,
             schema.getVersion());
-    this.refresh = clientBuilder.refresh;
     this.client = clientBuilder.build();
   }
 
@@ -108,7 +106,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
 
   @Override
   public void delete(K c) throws IOException {
-    Bulk bulk = addActions(new Bulk.Builder(), c).refresh(refresh).build();
+    Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build();
     JestResult result = client.execute(bulk);
     if (!result.isSucceeded()) {
       throw new IOException(

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java

@@ -96,7 +96,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
             .defaultIndex(indexName)
             .defaultType(ACCOUNTS)
             .addAction(insert(ACCOUNTS, as))
-            .refresh(refresh)
+            .refresh(true)
             .build();
     JestResult result = client.execute(bulk);
     if (!result.isSucceeded()) {

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java

@@ -137,7 +137,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
             .defaultType("changes")
             .addAction(insert(insertIndex, cd))
             .addAction(delete(deleteIndex, cd.getId()))
-            .refresh(refresh)
+            .refresh(true)
             .build();
     JestResult result = client.execute(bulk);
     if (!result.isSucceeded()) {

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java

@@ -0,0 +1,71 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch;
+
+import com.google.common.base.MoreObjects;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import org.eclipse.jgit.lib.Config;
+
+@Singleton
+class ElasticConfiguration {
+  private static final String DEFAULT_HOST = "localhost";
+  private static final String DEFAULT_PORT = "9200";
+  private static final String DEFAULT_PROTOCOL = "http";
+
+  final List<String> urls;
+
+  @Inject
+  ElasticConfiguration(@GerritServerConfig Config cfg) {
+    Set<String> subsections = cfg.getSubsections("elasticsearch");
+    if (subsections.isEmpty()) {
+      this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT));
+    } else {
+      this.urls = new ArrayList<>(subsections.size());
+      for (String subsection : subsections) {
+        String port = getString(cfg, subsection, "port", DEFAULT_PORT);
+        String host = getString(cfg, subsection, "hostname", DEFAULT_HOST);
+        String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL);
+        this.urls.add(buildUrl(protocol, host, port));
+      }
+    }
+  }
+
+  private String getString(Config cfg, String subsection, String name, String defaultValue) {
+    return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
+  }
+
+  private String buildUrl(String protocol, String hostname, String port) {
+    try {
+      return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
+    } catch (MalformedURLException | NumberFormatException e) {
+      throw new RuntimeException(
+          "Cannot build url to Elasticsearch from values: protocol="
+              + protocol
+              + " hostname="
+              + hostname
+              + " port="
+              + port,
+          e);
+    }
+  }
+}

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java

@@ -93,7 +93,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, A
             .defaultIndex(indexName)
             .defaultType(GROUPS)
             .addAction(insert(GROUPS, group))
-            .refresh(refresh)
+            .refresh(true)
             .build();
     JestResult result = client.execute(bulk);
     if (!result.isSucceeded()) {

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/JestClientBuilder.java

@@ -14,72 +14,30 @@
 
 package com.google.gerrit.elasticsearch;
 
-import com.google.common.base.MoreObjects;
-import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import io.searchbox.client.JestClientFactory;
 import io.searchbox.client.config.HttpClientConfig;
 import io.searchbox.client.http.JestHttpClient;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.concurrent.TimeUnit;
-import org.eclipse.jgit.lib.Config;
 
 @Singleton
 class JestClientBuilder {
-
-  private final String url;
-
-  final boolean refresh;
+  private final ElasticConfiguration cfg;
 
   @Inject
-  JestClientBuilder(@GerritServerConfig Config cfg) {
-    String protocol = MoreObjects.firstNonNull(cfg.getString("index", null, "protocol"), "http");
-    String hostname =
-        MoreObjects.firstNonNull(cfg.getString("index", null, "hostname"), "localhost");
-    String port = String.valueOf(cfg.getInt("index", null, "port", 9200));
-
-    // By default Elasticsearch has a 1s delay before changes are available in
-    // the index.  Setting refresh(true) on calls to the index makes the index
-    // refresh immediately.
-    //
-    // Discovery should be disabled during test mode to prevent spurious
-    // connection failures caused by the client starting up and being ready
-    // before the test node.
-    //
-    // This setting should only be set to true during testing, and is not
-    // documented.
-    this.refresh = cfg.getBoolean("index", "elasticsearch", "test", false);
-
-    this.url = buildUrl(protocol, hostname, port);
+  JestClientBuilder(ElasticConfiguration cfg) {
+    this.cfg = cfg;
   }
 
   JestHttpClient build() {
     JestClientFactory factory = new JestClientFactory();
     factory.setHttpClientConfig(
-        new HttpClientConfig.Builder(url)
+        new HttpClientConfig.Builder(cfg.urls)
             .multiThreaded(true)
-            // Temporary disable servers discovery.
-            // We can enable it again when we can wait for it to finish
             .discoveryEnabled(false)
             .discoveryFrequency(1L, TimeUnit.MINUTES)
             .build());
     return (JestHttpClient) factory.getObject();
   }
-
-  private String buildUrl(String protocol, String hostname, String port) {
-    try {
-      return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
-    } catch (MalformedURLException | NumberFormatException e) {
-      throw new RuntimeException(
-          "Cannot build url to Elasticsearch from values: protocol="
-              + protocol
-              + " hostname="
-              + hostname
-              + " port="
-              + port,
-          e);
-    }
-  }
 }