From 4ca46beb68f3addb2d76ebc2dfdb677cef892d9e Mon Sep 17 00:00:00 2001 From: Dariusz Luksza Date: Tue, 25 Apr 2017 11:40:29 +0200 Subject: [PATCH] ES: Allow to configure multiple ElasticSearch servers Add possibility to configure more than one ES servers in the gerrit.config file. We also permanently disable auto-discovery feature of JEST since administrators can configure multiple ES servers it is no longer required to relay on it. Especially that we would need to write complicated code to ensure that we don't use not fully initialized JEST client. Change-Id: I8277150ee6210183cb41a10ef179d7800471844b --- Documentation/config-gerrit.txt | 47 +++++++----- .../elasticsearch/AbstractElasticIndex.java | 4 +- .../elasticsearch/ElasticAccountIndex.java | 2 +- .../elasticsearch/ElasticChangeIndex.java | 2 +- .../elasticsearch/ElasticConfiguration.java | 71 +++++++++++++++++++ .../elasticsearch/ElasticGroupIndex.java | 2 +- .../elasticsearch/JestClientBuilder.java | 50 ++----------- 7 files changed, 108 insertions(+), 70 deletions(-) create mode 100644 gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index 72b55956c2..fcfa37c38f 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -2667,25 +2667,9 @@ Open and closed changes are indexed in a single index, separated into types 'open_changes' and 'closed_changes' respectively. The following settings are only used when the index type is -`ELASTICSEARCH`. +`ELASTICSEARCH`. To configure Elasticsearch servers look into +link:#elasticsearch[Elasticsearch section] -[[index.protocol]]index.protocol:: -+ -Elasticsearch server protocol [http|https]. -+ -Defaults to `http`. - -[[index.hostname]]index.hostname:: -+ -Elasticsearch server hostname. - -Defaults to `localhost`. - -[[index.port]]index.port:: -+ -Elasticsearch server port. -+ -Defaults to `9200`. [[index.prefix]]index.prefix:: + @@ -2695,6 +2679,33 @@ change index named 'gerrit1_changes_0001'. + Not set by default. +[[elasticsearch]] +=== Section elasticsearch + +WARNING: The Elasticsearch support is incomplete. Online reindexing +is not implemented yet. + +Each section correspond to the one Elasticsearch server. + +[[elasticsearch.name.protocol]]elasticsearch.name.protocol:: ++ +Elasticsearch server protocol [http|https]. ++ +Defaults to `http`. + +[[elasticsearch.name.hostname]]elasticsearch.name.hostname:: ++ +Elasticsearch server hostname. + +Defaults to `localhost`. + +[[elasticsearch.name.port]]elasticsearch.name.port:: ++ +Elasticsearch server port. ++ +Defaults to `9200`. + + [[ldap]] === Section ldap diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index a119c4aaa0..fbe77880de 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java @@ -63,7 +63,6 @@ abstract class AbstractElasticIndex implements Index { private final FillArgs fillArgs; private final SitePaths sitePaths; - protected final boolean refresh; protected final String indexName; protected final JestHttpClient client; protected final Gson gson; @@ -87,7 +86,6 @@ abstract class AbstractElasticIndex implements Index { Strings.nullToEmpty(cfg.getString("index", null, "prefix")), indexName, schema.getVersion()); - this.refresh = clientBuilder.refresh; this.client = clientBuilder.build(); } @@ -108,7 +106,7 @@ abstract class AbstractElasticIndex implements Index { @Override public void delete(K c) throws IOException { - Bulk bulk = addActions(new Bulk.Builder(), c).refresh(refresh).build(); + Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build(); JestResult result = client.execute(bulk); if (!result.isSucceeded()) { throw new IOException( diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java index 95e2b1ed68..2a46c40954 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java @@ -96,7 +96,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex .defaultType("changes") .addAction(insert(insertIndex, cd)) .addAction(delete(deleteIndex, cd.getId())) - .refresh(refresh) + .refresh(true) .build(); JestResult result = client.execute(bulk); if (!result.isSucceeded()) { diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java new file mode 100644 index 0000000000..1099f538f2 --- /dev/null +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java @@ -0,0 +1,71 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.common.base.MoreObjects; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import org.eclipse.jgit.lib.Config; + +@Singleton +class ElasticConfiguration { + private static final String DEFAULT_HOST = "localhost"; + private static final String DEFAULT_PORT = "9200"; + private static final String DEFAULT_PROTOCOL = "http"; + + final List urls; + + @Inject + ElasticConfiguration(@GerritServerConfig Config cfg) { + Set subsections = cfg.getSubsections("elasticsearch"); + if (subsections.isEmpty()) { + this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT)); + } else { + this.urls = new ArrayList<>(subsections.size()); + for (String subsection : subsections) { + String port = getString(cfg, subsection, "port", DEFAULT_PORT); + String host = getString(cfg, subsection, "hostname", DEFAULT_HOST); + String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL); + this.urls.add(buildUrl(protocol, host, port)); + } + } + } + + private String getString(Config cfg, String subsection, String name, String defaultValue) { + return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue); + } + + private String buildUrl(String protocol, String hostname, String port) { + try { + return new URL(protocol, hostname, Integer.parseInt(port), "").toString(); + } catch (MalformedURLException | NumberFormatException e) { + throw new RuntimeException( + "Cannot build url to Elasticsearch from values: protocol=" + + protocol + + " hostname=" + + hostname + + " port=" + + port, + e); + } + } +} diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java index 4bc8eeb547..e2c34f2bb3 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java @@ -93,7 +93,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex