ES: Allow to configure multiple ElasticSearch servers
Add possibility to configure more than one ES servers in the gerrit.config file. We also permanently disable auto-discovery feature of JEST since administrators can configure multiple ES servers it is no longer required to relay on it. Especially that we would need to write complicated code to ensure that we don't use not fully initialized JEST client. Change-Id: I8277150ee6210183cb41a10ef179d7800471844b
This commit is contained in:
parent
74bb6d6184
commit
4ca46beb68
@ -2667,25 +2667,9 @@ Open and closed changes are indexed in a single index, separated
|
|||||||
into types 'open_changes' and 'closed_changes' respectively.
|
into types 'open_changes' and 'closed_changes' respectively.
|
||||||
|
|
||||||
The following settings are only used when the index type is
|
The following settings are only used when the index type is
|
||||||
`ELASTICSEARCH`.
|
`ELASTICSEARCH`. To configure Elasticsearch servers look into
|
||||||
|
link:#elasticsearch[Elasticsearch section]
|
||||||
|
|
||||||
[[index.protocol]]index.protocol::
|
|
||||||
+
|
|
||||||
Elasticsearch server protocol [http|https].
|
|
||||||
+
|
|
||||||
Defaults to `http`.
|
|
||||||
|
|
||||||
[[index.hostname]]index.hostname::
|
|
||||||
+
|
|
||||||
Elasticsearch server hostname.
|
|
||||||
|
|
||||||
Defaults to `localhost`.
|
|
||||||
|
|
||||||
[[index.port]]index.port::
|
|
||||||
+
|
|
||||||
Elasticsearch server port.
|
|
||||||
+
|
|
||||||
Defaults to `9200`.
|
|
||||||
|
|
||||||
[[index.prefix]]index.prefix::
|
[[index.prefix]]index.prefix::
|
||||||
+
|
+
|
||||||
@ -2695,6 +2679,33 @@ change index named 'gerrit1_changes_0001'.
|
|||||||
+
|
+
|
||||||
Not set by default.
|
Not set by default.
|
||||||
|
|
||||||
|
[[elasticsearch]]
|
||||||
|
=== Section elasticsearch
|
||||||
|
|
||||||
|
WARNING: The Elasticsearch support is incomplete. Online reindexing
|
||||||
|
is not implemented yet.
|
||||||
|
|
||||||
|
Each section correspond to the one Elasticsearch server.
|
||||||
|
|
||||||
|
[[elasticsearch.name.protocol]]elasticsearch.name.protocol::
|
||||||
|
+
|
||||||
|
Elasticsearch server protocol [http|https].
|
||||||
|
+
|
||||||
|
Defaults to `http`.
|
||||||
|
|
||||||
|
[[elasticsearch.name.hostname]]elasticsearch.name.hostname::
|
||||||
|
+
|
||||||
|
Elasticsearch server hostname.
|
||||||
|
|
||||||
|
Defaults to `localhost`.
|
||||||
|
|
||||||
|
[[elasticsearch.name.port]]elasticsearch.name.port::
|
||||||
|
+
|
||||||
|
Elasticsearch server port.
|
||||||
|
+
|
||||||
|
Defaults to `9200`.
|
||||||
|
|
||||||
|
|
||||||
[[ldap]]
|
[[ldap]]
|
||||||
=== Section ldap
|
=== Section ldap
|
||||||
|
|
||||||
|
@ -63,7 +63,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
|
|||||||
private final FillArgs fillArgs;
|
private final FillArgs fillArgs;
|
||||||
private final SitePaths sitePaths;
|
private final SitePaths sitePaths;
|
||||||
|
|
||||||
protected final boolean refresh;
|
|
||||||
protected final String indexName;
|
protected final String indexName;
|
||||||
protected final JestHttpClient client;
|
protected final JestHttpClient client;
|
||||||
protected final Gson gson;
|
protected final Gson gson;
|
||||||
@ -87,7 +86,6 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
|
|||||||
Strings.nullToEmpty(cfg.getString("index", null, "prefix")),
|
Strings.nullToEmpty(cfg.getString("index", null, "prefix")),
|
||||||
indexName,
|
indexName,
|
||||||
schema.getVersion());
|
schema.getVersion());
|
||||||
this.refresh = clientBuilder.refresh;
|
|
||||||
this.client = clientBuilder.build();
|
this.client = clientBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -108,7 +106,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void delete(K c) throws IOException {
|
public void delete(K c) throws IOException {
|
||||||
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(refresh).build();
|
Bulk bulk = addActions(new Bulk.Builder(), c).refresh(true).build();
|
||||||
JestResult result = client.execute(bulk);
|
JestResult result = client.execute(bulk);
|
||||||
if (!result.isSucceeded()) {
|
if (!result.isSucceeded()) {
|
||||||
throw new IOException(
|
throw new IOException(
|
||||||
|
@ -96,7 +96,7 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
|
|||||||
.defaultIndex(indexName)
|
.defaultIndex(indexName)
|
||||||
.defaultType(ACCOUNTS)
|
.defaultType(ACCOUNTS)
|
||||||
.addAction(insert(ACCOUNTS, as))
|
.addAction(insert(ACCOUNTS, as))
|
||||||
.refresh(refresh)
|
.refresh(true)
|
||||||
.build();
|
.build();
|
||||||
JestResult result = client.execute(bulk);
|
JestResult result = client.execute(bulk);
|
||||||
if (!result.isSucceeded()) {
|
if (!result.isSucceeded()) {
|
||||||
|
@ -137,7 +137,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
|
|||||||
.defaultType("changes")
|
.defaultType("changes")
|
||||||
.addAction(insert(insertIndex, cd))
|
.addAction(insert(insertIndex, cd))
|
||||||
.addAction(delete(deleteIndex, cd.getId()))
|
.addAction(delete(deleteIndex, cd.getId()))
|
||||||
.refresh(refresh)
|
.refresh(true)
|
||||||
.build();
|
.build();
|
||||||
JestResult result = client.execute(bulk);
|
JestResult result = client.execute(bulk);
|
||||||
if (!result.isSucceeded()) {
|
if (!result.isSucceeded()) {
|
||||||
|
@ -0,0 +1,71 @@
|
|||||||
|
// Copyright (C) 2017 The Android Open Source Project
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package com.google.gerrit.elasticsearch;
|
||||||
|
|
||||||
|
import com.google.common.base.MoreObjects;
|
||||||
|
import com.google.gerrit.server.config.GerritServerConfig;
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import com.google.inject.Singleton;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
import org.eclipse.jgit.lib.Config;
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
class ElasticConfiguration {
|
||||||
|
private static final String DEFAULT_HOST = "localhost";
|
||||||
|
private static final String DEFAULT_PORT = "9200";
|
||||||
|
private static final String DEFAULT_PROTOCOL = "http";
|
||||||
|
|
||||||
|
final List<String> urls;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
ElasticConfiguration(@GerritServerConfig Config cfg) {
|
||||||
|
Set<String> subsections = cfg.getSubsections("elasticsearch");
|
||||||
|
if (subsections.isEmpty()) {
|
||||||
|
this.urls = Arrays.asList(buildUrl(DEFAULT_PROTOCOL, DEFAULT_HOST, DEFAULT_PORT));
|
||||||
|
} else {
|
||||||
|
this.urls = new ArrayList<>(subsections.size());
|
||||||
|
for (String subsection : subsections) {
|
||||||
|
String port = getString(cfg, subsection, "port", DEFAULT_PORT);
|
||||||
|
String host = getString(cfg, subsection, "hostname", DEFAULT_HOST);
|
||||||
|
String protocol = getString(cfg, subsection, "protocol", DEFAULT_PROTOCOL);
|
||||||
|
this.urls.add(buildUrl(protocol, host, port));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getString(Config cfg, String subsection, String name, String defaultValue) {
|
||||||
|
return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildUrl(String protocol, String hostname, String port) {
|
||||||
|
try {
|
||||||
|
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
|
||||||
|
} catch (MalformedURLException | NumberFormatException e) {
|
||||||
|
throw new RuntimeException(
|
||||||
|
"Cannot build url to Elasticsearch from values: protocol="
|
||||||
|
+ protocol
|
||||||
|
+ " hostname="
|
||||||
|
+ hostname
|
||||||
|
+ " port="
|
||||||
|
+ port,
|
||||||
|
e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -93,7 +93,7 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, A
|
|||||||
.defaultIndex(indexName)
|
.defaultIndex(indexName)
|
||||||
.defaultType(GROUPS)
|
.defaultType(GROUPS)
|
||||||
.addAction(insert(GROUPS, group))
|
.addAction(insert(GROUPS, group))
|
||||||
.refresh(refresh)
|
.refresh(true)
|
||||||
.build();
|
.build();
|
||||||
JestResult result = client.execute(bulk);
|
JestResult result = client.execute(bulk);
|
||||||
if (!result.isSucceeded()) {
|
if (!result.isSucceeded()) {
|
||||||
|
@ -14,72 +14,30 @@
|
|||||||
|
|
||||||
package com.google.gerrit.elasticsearch;
|
package com.google.gerrit.elasticsearch;
|
||||||
|
|
||||||
import com.google.common.base.MoreObjects;
|
|
||||||
import com.google.gerrit.server.config.GerritServerConfig;
|
|
||||||
import com.google.inject.Inject;
|
import com.google.inject.Inject;
|
||||||
import com.google.inject.Singleton;
|
import com.google.inject.Singleton;
|
||||||
import io.searchbox.client.JestClientFactory;
|
import io.searchbox.client.JestClientFactory;
|
||||||
import io.searchbox.client.config.HttpClientConfig;
|
import io.searchbox.client.config.HttpClientConfig;
|
||||||
import io.searchbox.client.http.JestHttpClient;
|
import io.searchbox.client.http.JestHttpClient;
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import org.eclipse.jgit.lib.Config;
|
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
class JestClientBuilder {
|
class JestClientBuilder {
|
||||||
|
private final ElasticConfiguration cfg;
|
||||||
private final String url;
|
|
||||||
|
|
||||||
final boolean refresh;
|
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
JestClientBuilder(@GerritServerConfig Config cfg) {
|
JestClientBuilder(ElasticConfiguration cfg) {
|
||||||
String protocol = MoreObjects.firstNonNull(cfg.getString("index", null, "protocol"), "http");
|
this.cfg = cfg;
|
||||||
String hostname =
|
|
||||||
MoreObjects.firstNonNull(cfg.getString("index", null, "hostname"), "localhost");
|
|
||||||
String port = String.valueOf(cfg.getInt("index", null, "port", 9200));
|
|
||||||
|
|
||||||
// By default Elasticsearch has a 1s delay before changes are available in
|
|
||||||
// the index. Setting refresh(true) on calls to the index makes the index
|
|
||||||
// refresh immediately.
|
|
||||||
//
|
|
||||||
// Discovery should be disabled during test mode to prevent spurious
|
|
||||||
// connection failures caused by the client starting up and being ready
|
|
||||||
// before the test node.
|
|
||||||
//
|
|
||||||
// This setting should only be set to true during testing, and is not
|
|
||||||
// documented.
|
|
||||||
this.refresh = cfg.getBoolean("index", "elasticsearch", "test", false);
|
|
||||||
|
|
||||||
this.url = buildUrl(protocol, hostname, port);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
JestHttpClient build() {
|
JestHttpClient build() {
|
||||||
JestClientFactory factory = new JestClientFactory();
|
JestClientFactory factory = new JestClientFactory();
|
||||||
factory.setHttpClientConfig(
|
factory.setHttpClientConfig(
|
||||||
new HttpClientConfig.Builder(url)
|
new HttpClientConfig.Builder(cfg.urls)
|
||||||
.multiThreaded(true)
|
.multiThreaded(true)
|
||||||
// Temporary disable servers discovery.
|
|
||||||
// We can enable it again when we can wait for it to finish
|
|
||||||
.discoveryEnabled(false)
|
.discoveryEnabled(false)
|
||||||
.discoveryFrequency(1L, TimeUnit.MINUTES)
|
.discoveryFrequency(1L, TimeUnit.MINUTES)
|
||||||
.build());
|
.build());
|
||||||
return (JestHttpClient) factory.getObject();
|
return (JestHttpClient) factory.getObject();
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildUrl(String protocol, String hostname, String port) {
|
|
||||||
try {
|
|
||||||
return new URL(protocol, hostname, Integer.parseInt(port), "").toString();
|
|
||||||
} catch (MalformedURLException | NumberFormatException e) {
|
|
||||||
throw new RuntimeException(
|
|
||||||
"Cannot build url to Elasticsearch from values: protocol="
|
|
||||||
+ protocol
|
|
||||||
+ " hostname="
|
|
||||||
+ hostname
|
|
||||||
+ " port="
|
|
||||||
+ port,
|
|
||||||
e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user