Merge "Merge branch 'stable-2.15'"

This commit is contained in:
David Pursehouse 2018-06-05 11:02:06 +00:00 committed by Gerrit Code Review
commit b9999b9a52
39 changed files with 627 additions and 169 deletions

View File

@ -0,0 +1,27 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
class ElasticException extends RuntimeException {
private static final long serialVersionUID = 1L;
ElasticException(String message) {
super(message);
}
ElasticException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@ -0,0 +1,143 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
class ElasticRestClientProvider implements Provider<RestClient>, LifecycleListener {
private static final Logger log = LoggerFactory.getLogger(ElasticRestClientProvider.class);
private final HttpHost[] hosts;
private final String username;
private final String password;
private RestClient client;
@Inject
ElasticRestClientProvider(ElasticConfiguration cfg) {
hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]);
username = cfg.username;
password = cfg.password;
}
public static LifecycleModule module() {
return new LifecycleModule() {
@Override
protected void configure() {
listener().to(ElasticRestClientProvider.class);
}
};
}
@Override
public RestClient get() {
if (client == null) {
synchronized (this) {
if (client == null) {
client = build();
ElasticVersion version = getVersion();
log.info("Elasticsearch integration version {}", version);
}
}
}
return client;
}
@Override
public void start() {}
@Override
public void stop() {
if (client != null) {
try {
client.close();
} catch (IOException e) {
// Ignore. We can't do anything about it.
}
}
}
public static class FailedToGetVersion extends ElasticException {
private static final long serialVersionUID = 1L;
private static final String MESSAGE = "Failed to get Elasticsearch version";
FailedToGetVersion(StatusLine status) {
super(String.format("%s: %d %s", MESSAGE, status.getStatusCode(), status.getReasonPhrase()));
}
FailedToGetVersion(Throwable cause) {
super(MESSAGE, cause);
}
}
private ElasticVersion getVersion() throws ElasticException {
try {
Response response = client.performRequest("GET", "");
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
throw new FailedToGetVersion(statusLine);
}
String version =
new JsonParser()
.parse(AbstractElasticIndex.getContent(response))
.getAsJsonObject()
.get("version")
.getAsJsonObject()
.get("number")
.getAsString();
log.info("Connected to Elasticsearch version {}", version);
return ElasticVersion.forVersion(version);
} catch (IOException e) {
throw new FailedToGetVersion(e);
}
}
private RestClient build() {
RestClientBuilder builder = RestClient.builder(hosts);
setConfiguredCredentialsIfAny(builder);
return builder.build();
}
private void setConfiguredCredentialsIfAny(RestClientBuilder builder) {
if (username != null && password != null) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(
(HttpAsyncClientBuilder httpClientBuilder) ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
}
}

View File

@ -0,0 +1,60 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.base.Joiner;
import java.util.regex.Pattern;
public enum ElasticVersion {
V2_4("2.4.*"),
V5_6("5.6.*"),
V6_2("6.2.*");
private final String version;
private final Pattern pattern;
private ElasticVersion(String version) {
this.version = version;
this.pattern = Pattern.compile(version);
}
public static class InvalidVersion extends ElasticException {
private static final long serialVersionUID = 1L;
InvalidVersion(String version) {
super(
String.format(
"Invalid version: [%s]. Supported versions: %s", version, supportedVersions()));
}
}
public static ElasticVersion forVersion(String version) throws InvalidVersion {
for (ElasticVersion value : ElasticVersion.values()) {
if (value.pattern.matcher(version).matches()) {
return value;
}
}
throw new InvalidVersion(version);
}
public static String supportedVersions() {
return Joiner.on(", ").join(ElasticVersion.values());
}
@Override
public String toString() {
return version;
}
}

View File

@ -0,0 +1,45 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import static com.google.common.truth.Truth.assertThat;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class ElasticVersionTest {
@Rule public ExpectedException exception = ExpectedException.none();
@Test
public void supportedVersion() throws Exception {
assertThat(ElasticVersion.forVersion("2.4.0")).isEqualTo(ElasticVersion.V2_4);
assertThat(ElasticVersion.forVersion("2.4.6")).isEqualTo(ElasticVersion.V2_4);
assertThat(ElasticVersion.forVersion("5.6.0")).isEqualTo(ElasticVersion.V5_6);
assertThat(ElasticVersion.forVersion("5.6.9")).isEqualTo(ElasticVersion.V5_6);
assertThat(ElasticVersion.forVersion("6.2.0")).isEqualTo(ElasticVersion.V6_2);
assertThat(ElasticVersion.forVersion("6.2.4")).isEqualTo(ElasticVersion.V6_2);
}
@Test
public void unsupportedVersion() throws Exception {
exception.expect(ElasticVersion.InvalidVersion.class);
exception.expectMessage(
"Invalid version: [4.0.0]. Supported versions: " + ElasticVersion.supportedVersions());
ElasticVersion.forVersion("4.0.0");
}
}

View File

@ -19,7 +19,6 @@ import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ListMultimap;
@ -37,7 +36,6 @@ import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gson.Gson;
@ -70,9 +68,7 @@ import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -110,30 +106,25 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
private final RestClient client;
private final ElasticRestClientProvider client;
protected final String indexName;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
AbstractElasticIndex(
@GerritServerConfig Config cfg,
ElasticConfiguration cfg,
SitePaths sitePaths,
Schema<V> schema,
ElasticRestClientBuilder clientBuilder,
ElasticRestClientProvider client,
String indexName) {
this.sitePaths = sitePaths;
this.schema = schema;
this.gson = new GsonBuilder().setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES).create();
this.queryBuilder = new ElasticQueryBuilder();
this.indexName =
String.format(
"%s%s_%04d",
Strings.nullToEmpty(cfg.getString("elasticsearch", null, "prefix")),
indexName,
schema.getVersion());
this.indexName = cfg.getIndexName(indexName, schema.getVersion());
this.indexNameRaw = indexName;
this.client = clientBuilder.build();
this.client = client;
}
@Override
@ -143,11 +134,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void close() {
try {
client.close();
} catch (IOException e) {
// Ignored.
}
// Do nothing. Client is closed by the provider.
}
@Override
@ -169,10 +156,10 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
@Override
public void deleteAll() throws IOException {
// Delete the index, if it exists.
Response response = client.performRequest("HEAD", indexName);
Response response = client.get().performRequest("HEAD", indexName);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
response = client.performRequest("DELETE", indexName);
response = client.get().performRequest("DELETE", indexName);
statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IOException(
@ -286,7 +273,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
String method, Object payload, String uri, Map<String, String> params) throws IOException {
String payloadStr = payload instanceof String ? (String) payload : payload.toString();
HttpEntity entity = new NStringEntity(payloadStr, ContentType.APPLICATION_JSON);
return client.performRequest(method, uri, params, entity);
return client.get().performRequest(method, uri, params, entity);
}
protected class ElasticQuerySource implements DataSource<V> {

View File

@ -29,7 +29,6 @@ import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountField;
@ -43,7 +42,6 @@ import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
@ -64,12 +62,12 @@ public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, Accoun
@Inject
ElasticAccountIndex(
@GerritServerConfig Config cfg,
ElasticConfiguration cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
ElasticRestClientBuilder clientBuilder,
ElasticRestClientProvider client,
@Assisted Schema<AccountState> schema) {
super(cfg, sitePaths, schema, clientBuilder, ACCOUNTS);
super(cfg, sitePaths, schema, client, ACCOUNTS);
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema);
this.schema = schema;

View File

@ -49,7 +49,6 @@ import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.ReviewerByEmailSet;
import com.google.gerrit.server.ReviewerSet;
import com.google.gerrit.server.StarredChangesUtil;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
@ -71,7 +70,6 @@ import java.util.Optional;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
/** Secondary index implementation using Elasticsearch. */
@ -99,11 +97,11 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
@Inject
ElasticChangeIndex(
@GerritServerConfig Config cfg,
ElasticConfiguration cfg,
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
SitePaths sitePaths,
ElasticRestClientBuilder clientBuilder,
ElasticRestClientProvider clientBuilder,
@Assisted Schema<ChangeData> schema) {
super(cfg, sitePaths, schema, clientBuilder, CHANGES);
this.db = db;

View File

@ -15,6 +15,7 @@
package com.google.gerrit.elasticsearch;
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@ -32,6 +33,8 @@ class ElasticConfiguration {
private static final String DEFAULT_PORT = "9200";
private static final String DEFAULT_PROTOCOL = "http";
private final Config cfg;
final List<HttpHost> urls;
final String username;
final String password;
@ -41,9 +44,11 @@ class ElasticConfiguration {
final TimeUnit maxConnectionIdleUnit = TimeUnit.MILLISECONDS;
final int maxTotalConnection;
final int readTimeout;
final String prefix;
@Inject
ElasticConfiguration(@GerritServerConfig Config cfg) {
this.cfg = cfg;
this.username = cfg.getString("elasticsearch", null, "username");
this.password = cfg.getString("elasticsearch", null, "password");
this.requestCompression = cfg.getBoolean("elasticsearch", null, "requestCompression", false);
@ -55,6 +60,7 @@ class ElasticConfiguration {
this.maxTotalConnection = cfg.getInt("elasticsearch", null, "maxTotalConnection", 1);
this.readTimeout =
(int) cfg.getTimeUnit("elasticsearch", null, "readTimeout", 3000, TimeUnit.MICROSECONDS);
this.prefix = Strings.nullToEmpty(cfg.getString("elasticsearch", null, "prefix"));
Set<String> subsections = cfg.getSubsections("elasticsearch");
if (subsections.isEmpty()) {
@ -74,6 +80,14 @@ class ElasticConfiguration {
}
}
public Config getConfig() {
return cfg;
}
public String getIndexName(String name, int schemaVersion) {
return String.format("%s%s_%04d", prefix, name, schemaVersion);
}
private String getString(Config cfg, String subsection, String name, String defaultValue) {
return MoreObjects.firstNonNull(cfg.getString("elasticsearch", subsection, name), defaultValue);
}

View File

@ -0,0 +1,27 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
class ElasticException extends RuntimeException {
private static final long serialVersionUID = 1L;
ElasticException(String message) {
super(message);
}
ElasticException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@ -26,7 +26,6 @@ import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.server.account.GroupCache;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.index.IndexUtils;
@ -41,7 +40,6 @@ import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, InternalGroup>
@ -62,12 +60,12 @@ public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, I
@Inject
ElasticGroupIndex(
@GerritServerConfig Config cfg,
ElasticConfiguration cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
ElasticRestClientBuilder clientBuilder,
ElasticRestClientProvider client,
@Assisted Schema<InternalGroup> schema) {
super(cfg, sitePaths, schema, clientBuilder, GROUPS);
super(cfg, sitePaths, schema, client, GROUPS);
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema);
this.schema = schema;

View File

@ -41,6 +41,12 @@ public class ElasticIndexModule extends AbstractIndexModule {
super(singleVersions, threads, onlineUpgrade, slave);
}
@Override
public void configure() {
super.configure();
install(ElasticRestClientProvider.module());
}
@Override
protected Class<? extends AccountIndex> getAccountIndex() {
return ElasticAccountIndex.class;
@ -63,6 +69,6 @@ public class ElasticIndexModule extends AbstractIndexModule {
@Override
protected Class<? extends VersionManager> getVersionManager() {
return ElasticVersionManager.class;
return ElasticIndexVersionManager.class;
}
}

View File

@ -25,20 +25,19 @@ import java.util.List;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
@Singleton
class ElasticIndexVersionDiscovery {
private final RestClient client;
private final ElasticRestClientProvider client;
@Inject
ElasticIndexVersionDiscovery(ElasticRestClientBuilder clientBuilder) {
this.client = clientBuilder.build();
ElasticIndexVersionDiscovery(ElasticRestClientProvider client) {
this.client = client;
}
List<String> discover(String prefix, String indexName) throws IOException {
String name = prefix + indexName + "_";
Response response = client.performRequest(HttpGet.METHOD_NAME, name + "*/_aliases");
Response response = client.get().performRequest(HttpGet.METHOD_NAME, name + "*/_aliases");
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
return new JsonParser()

View File

@ -35,14 +35,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class ElasticVersionManager extends VersionManager {
private static final Logger log = LoggerFactory.getLogger(ElasticVersionManager.class);
public class ElasticIndexVersionManager extends VersionManager {
private static final Logger log = LoggerFactory.getLogger(ElasticIndexVersionManager.class);
private final String prefix;
private final ElasticIndexVersionDiscovery versionDiscovery;
@Inject
ElasticVersionManager(
ElasticIndexVersionManager(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
DynamicSet<OnlineUpgradeListener> listeners,

View File

@ -26,7 +26,7 @@ class ElasticMapping {
for (FieldDef<?, ?> field : schema.getFields().values()) {
String name = field.getName();
FieldType<?> fieldType = field.getType();
if (fieldType == FieldType.EXACT) {
if (fieldType == FieldType.EXACT || fieldType == FieldType.KEYWORD) {
mapping.addExactField(name);
} else if (fieldType == FieldType.TIMESTAMP) {
mapping.addTimestamp(name);

View File

@ -28,7 +28,6 @@ import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.project.ProjectCache;
@ -41,7 +40,6 @@ import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.client.Response;
public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, ProjectData>
@ -62,10 +60,10 @@ public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, P
@Inject
ElasticProjectIndex(
@GerritServerConfig Config cfg,
ElasticConfiguration cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
ElasticRestClientBuilder clientBuilder,
ElasticRestClientProvider clientBuilder,
@Assisted Schema<ProjectData> schema) {
super(cfg, sitePaths, schema, clientBuilder, PROJECTS);
this.projectCache = projectCache;

View File

@ -94,7 +94,7 @@ public class ElasticQueryBuilder {
return intRangeQuery(p);
} else if (type == FieldType.TIMESTAMP) {
return timestampQuery(p);
} else if (type == FieldType.EXACT) {
} else if (type == FieldType.EXACT || type == FieldType.KEYWORD) {
return exactQuery(p);
} else if (type == FieldType.PREFIX) {
return QueryBuilders.matchPhrasePrefixQuery(name, value);

View File

@ -1,58 +0,0 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
@Singleton
class ElasticRestClientBuilder {
private final HttpHost[] hosts;
private final String username;
private final String password;
@Inject
ElasticRestClientBuilder(ElasticConfiguration cfg) {
hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]);
username = cfg.username;
password = cfg.password;
}
RestClient build() {
RestClientBuilder builder = RestClient.builder(hosts);
setConfiguredCredentialsIfAny(builder);
return builder.build();
}
private void setConfiguredCredentialsIfAny(RestClientBuilder builder) {
if (username != null && password != null) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(
(HttpAsyncClientBuilder httpClientBuilder) ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
}
}

View File

@ -0,0 +1,143 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
class ElasticRestClientProvider implements Provider<RestClient>, LifecycleListener {
private static final Logger log = LoggerFactory.getLogger(ElasticRestClientProvider.class);
private final HttpHost[] hosts;
private final String username;
private final String password;
private RestClient client;
@Inject
ElasticRestClientProvider(ElasticConfiguration cfg) {
hosts = cfg.urls.toArray(new HttpHost[cfg.urls.size()]);
username = cfg.username;
password = cfg.password;
}
public static LifecycleModule module() {
return new LifecycleModule() {
@Override
protected void configure() {
listener().to(ElasticRestClientProvider.class);
}
};
}
@Override
public RestClient get() {
if (client == null) {
synchronized (this) {
if (client == null) {
client = build();
ElasticVersion version = getVersion();
log.info("Elasticsearch integration version {}", version);
}
}
}
return client;
}
@Override
public void start() {}
@Override
public void stop() {
if (client != null) {
try {
client.close();
} catch (IOException e) {
// Ignore. We can't do anything about it.
}
}
}
public static class FailedToGetVersion extends ElasticException {
private static final long serialVersionUID = 1L;
private static final String MESSAGE = "Failed to get Elasticsearch version";
FailedToGetVersion(StatusLine status) {
super(String.format("%s: %d %s", MESSAGE, status.getStatusCode(), status.getReasonPhrase()));
}
FailedToGetVersion(Throwable cause) {
super(MESSAGE, cause);
}
}
private ElasticVersion getVersion() throws ElasticException {
try {
Response response = client.performRequest("GET", "");
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
throw new FailedToGetVersion(statusLine);
}
String version =
new JsonParser()
.parse(AbstractElasticIndex.getContent(response))
.getAsJsonObject()
.get("version")
.getAsJsonObject()
.get("number")
.getAsString();
log.info("Connected to Elasticsearch version {}", version);
return ElasticVersion.forVersion(version);
} catch (IOException e) {
throw new FailedToGetVersion(e);
}
}
private RestClient build() {
RestClientBuilder builder = RestClient.builder(hosts);
setConfiguredCredentialsIfAny(builder);
return builder.build();
}
private void setConfiguredCredentialsIfAny(RestClientBuilder builder) {
if (username != null && password != null) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(
(HttpAsyncClientBuilder httpClientBuilder) ->
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
}
}

View File

@ -0,0 +1,60 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.base.Joiner;
import java.util.regex.Pattern;
public enum ElasticVersion {
V2_4("2.4.*"),
V5_6("5.6.*"),
V6_2("6.2.*");
private final String version;
private final Pattern pattern;
private ElasticVersion(String version) {
this.version = version;
this.pattern = Pattern.compile(version);
}
public static class InvalidVersion extends ElasticException {
private static final long serialVersionUID = 1L;
InvalidVersion(String version) {
super(
String.format(
"Invalid version: [%s]. Supported versions: %s", version, supportedVersions()));
}
}
public static ElasticVersion forVersion(String version) throws InvalidVersion {
for (ElasticVersion value : ElasticVersion.values()) {
if (value.pattern.matcher(version).matches()) {
return value;
}
}
throw new InvalidVersion(version);
}
public static String supportedVersions() {
return Joiner.on(", ").join(ElasticVersion.values());
}
@Override
public String toString() {
return version;
}
}

View File

@ -19,8 +19,9 @@ import java.util.ArrayList;
import java.util.List;
/**
* A Query that matches documents matching boolean combinations of other queries. A trimmed down
* version of org.elasticsearch.index.query.BoolQueryBuilder for this very package.
* A Query that matches documents matching boolean combinations of other queries.
*
* <p>A trimmed down version of org.elasticsearch.index.query.BoolQueryBuilder.
*/
public class BoolQueryBuilder extends QueryBuilder {

View File

@ -17,8 +17,9 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* Constructs a query that only match on documents that the field has a value in them. A trimmed
* down version of org.elasticsearch.index.query.ExistsQueryBuilder for this very package.
* Constructs a query that only match on documents that the field has a value in them.
*
* <p>A trimmed down version of org.elasticsearch.index.query.ExistsQueryBuilder.
*/
class ExistsQueryBuilder extends QueryBuilder {

View File

@ -17,8 +17,9 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A query that matches on all documents. A trimmed down version of
* org.elasticsearch.index.query.MatchAllQueryBuilder for this very package.
* A query that matches on all documents.
*
* <p>A trimmed down version of org.elasticsearch.index.query.MatchAllQueryBuilder.
*/
class MatchAllQueryBuilder extends QueryBuilder {

View File

@ -19,8 +19,9 @@ import java.util.Locale;
/**
* Match query is a query that analyzes the text and constructs a query as the result of the
* analysis. It can construct different queries based on the type provided. A trimmed down version
* of org.elasticsearch.index.query.MatchQueryBuilder for this very package.
* analysis. It can construct different queries based on the type provided.
*
* <p>A trimmed down version of org.elasticsearch.index.query.MatchQueryBuilder.
*/
class MatchQueryBuilder extends QueryBuilder {

View File

@ -16,7 +16,7 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/** A trimmed down version of org.elasticsearch.index.query.QueryBuilder for this very package. */
/** A trimmed down version of org.elasticsearch.index.query.QueryBuilder. */
public abstract class QueryBuilder {
protected QueryBuilder() {}

View File

@ -15,8 +15,9 @@
package com.google.gerrit.elasticsearch.builders;
/**
* A static factory for simple "import static" usage. A trimmed down version of
* org.elasticsearch.index.query.QueryBuilders for this very package.
* A static factory for simple "import static" usage.
*
* <p>A trimmed down version of org.elasticsearch.index.query.QueryBuilders.
*/
public abstract class QueryBuilders {

View File

@ -16,10 +16,7 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A trimmed down and further altered version of org.elasticsearch.action.support.QuerySourceBuilder
* for this very package.
*/
/** A trimmed down and modified version of org.elasticsearch.action.support.QuerySourceBuilder. */
class QuerySourceBuilder {
private final QueryBuilder queryBuilder;

View File

@ -17,8 +17,9 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents within an range of terms. A trimmed down version of
* org.elasticsearch.index.query.RangeQueryBuilder for this very package.
* A Query that matches documents within an range of terms.
*
* <p>A trimmed down version of org.elasticsearch.index.query.RangeQueryBuilder.
*/
public class RangeQueryBuilder extends QueryBuilder {

View File

@ -17,8 +17,9 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that does fuzzy matching for a specific value. A trimmed down version of
* org.elasticsearch.index.query.RegexpQueryBuilder for this very package.
* A Query that does fuzzy matching for a specific value.
*
* <p>A trimmed down version of org.elasticsearch.index.query.RegexpQueryBuilder.
*/
class RegexpQueryBuilder extends QueryBuilder {

View File

@ -18,8 +18,9 @@ import java.io.IOException;
import java.util.List;
/**
* A search source builder allowing to easily build search source. A trimmed down and further
* altered version of org.elasticsearch.search.builder.SearchSourceBuilder for this very package.
* A search source builder allowing to easily build search source.
*
* <p>A trimmed down and modified version of org.elasticsearch.search.builder.SearchSourceBuilder.
*/
public class SearchSourceBuilder {

View File

@ -17,8 +17,9 @@ package com.google.gerrit.elasticsearch.builders;
import java.io.IOException;
/**
* A Query that matches documents containing a term. A trimmed down version of
* org.elasticsearch.index.query.TermQueryBuilder for this very package.
* A Query that matches documents containing a term.
*
* <p>A trimmed down version of org.elasticsearch.index.query.TermQueryBuilder.
*/
class TermQueryBuilder extends QueryBuilder {

View File

@ -26,10 +26,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Date;
/**
* A trimmed down and further altered version of org.elasticsearch.common.xcontent.XContentBuilder
* for this very package.
*/
/** A trimmed down and modified version of org.elasticsearch.common.xcontent.XContentBuilder. */
public final class XContentBuilder implements Closeable {
private final JsonGenerator generator;

View File

@ -34,6 +34,10 @@ public final class FieldDef<I, T> {
return new FieldDef.Builder<>(FieldType.EXACT, name);
}
public static FieldDef.Builder<String> keyword(String name) {
return new FieldDef.Builder<>(FieldType.KEYWORD, name);
}
public static FieldDef.Builder<String> fullText(String name) {
return new FieldDef.Builder<>(FieldType.FULL_TEXT, name);
}

View File

@ -33,6 +33,9 @@ public class FieldType<T> {
/** A string field searched using exact-match semantics. */
public static final FieldType<String> EXACT = new FieldType<>("EXACT");
/** A Keyword field searched using non-analyzed-match semantics. */
public static final FieldType<String> KEYWORD = new FieldType<>("KEYWORD");
/** A string field searched using prefix. */
public static final FieldType<String> PREFIX = new FieldType<>("PREFIX");

View File

@ -332,7 +332,7 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
} else if (type == FieldType.EXACT || type == FieldType.PREFIX) {
} else if (type == FieldType.KEYWORD || type == FieldType.EXACT || type == FieldType.PREFIX) {
for (Object value : values.getValues()) {
doc.add(new StringField(name, (String) value, store));
}
@ -355,7 +355,10 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
for (IndexableField field : doc.getFields()) {
checkArgument(allFields.containsKey(field.name()), "Unrecognized field " + field.name());
FieldType<?> type = allFields.get(field.name()).getType();
if (type == FieldType.EXACT || type == FieldType.FULL_TEXT || type == FieldType.PREFIX) {
if (type == FieldType.EXACT
|| type == FieldType.FULL_TEXT
|| type == FieldType.PREFIX
|| type == FieldType.KEYWORD) {
rawFields.put(field.name(), field.stringValue());
} else if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
rawFields.put(field.name(), field.numericValue().intValue());

View File

@ -141,20 +141,21 @@ public class QueryBuilder<V> {
"field not in schema v%s: %s",
schema.getVersion(),
p.getField().getName());
if (p.getType() == FieldType.INTEGER) {
FieldType<?> type = p.getType();
if (type == FieldType.INTEGER) {
return intQuery(p);
} else if (p.getType() == FieldType.INTEGER_RANGE) {
} else if (type == FieldType.INTEGER_RANGE) {
return intRangeQuery(p);
} else if (p.getType() == FieldType.TIMESTAMP) {
} else if (type == FieldType.TIMESTAMP) {
return timestampQuery(p);
} else if (p.getType() == FieldType.EXACT) {
} else if (type == FieldType.EXACT || type == FieldType.KEYWORD) {
return exactQuery(p);
} else if (p.getType() == FieldType.PREFIX) {
} else if (type == FieldType.PREFIX) {
return prefixQuery(p);
} else if (p.getType() == FieldType.FULL_TEXT) {
} else if (type == FieldType.FULL_TEXT) {
return fullTextQuery(p);
} else {
throw FieldType.badFieldType(p.getType());
throw FieldType.badFieldType(type);
}
}

View File

@ -106,12 +106,13 @@ public class WorkQueue {
}
/** Create a new executor queue. */
public ScheduledExecutorService createQueue(int poolsize, String prefix) {
return createQueue(poolsize, prefix, Thread.NORM_PRIORITY);
public ScheduledExecutorService createQueue(int poolsize, String queueName) {
return createQueue(poolsize, queueName, Thread.NORM_PRIORITY);
}
public ScheduledThreadPoolExecutor createQueue(int poolsize, String prefix, int threadPriority) {
Executor executor = new Executor(poolsize, prefix);
public ScheduledThreadPoolExecutor createQueue(
int poolsize, String queueName, int threadPriority) {
Executor executor = new Executor(poolsize, queueName);
executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(true);
queues.add(executor);
@ -201,7 +202,7 @@ public class WorkQueue {
private final ConcurrentHashMap<Integer, Task<?>> all;
private final String queueName;
Executor(int corePoolSize, String prefix) {
Executor(int corePoolSize, final String queueName) {
super(
corePoolSize,
new ThreadFactory() {
@ -211,7 +212,7 @@ public class WorkQueue {
@Override
public Thread newThread(Runnable task) {
final Thread t = parent.newThread(task);
t.setName(prefix + "-" + tid.getAndIncrement());
t.setName(queueName + "-" + tid.getAndIncrement());
t.setUncaughtExceptionHandler(LOG_UNCAUGHT_EXCEPTION);
return t;
}
@ -223,7 +224,7 @@ public class WorkQueue {
0.75f, // load factor
corePoolSize + 4 // concurrency level
);
queueName = prefix;
this.queueName = queueName;
}
@Override

View File

@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.gerrit.index.FieldDef.exact;
import static com.google.gerrit.index.FieldDef.fullText;
import static com.google.gerrit.index.FieldDef.integer;
import static com.google.gerrit.index.FieldDef.keyword;
import static com.google.gerrit.index.FieldDef.prefix;
import static com.google.gerrit.index.FieldDef.storedOnly;
import static com.google.gerrit.index.FieldDef.timestamp;
@ -40,11 +41,11 @@ public class GroupField {
/** Group UUID. */
public static final FieldDef<InternalGroup, String> UUID =
exact("uuid").stored().build(g -> g.getGroupUUID().get());
keyword("uuid").stored().build(g -> g.getGroupUUID().get());
/** Group owner UUID. */
public static final FieldDef<InternalGroup, String> OWNER_UUID =
exact("owner_uuid").build(g -> g.getOwnerGroupUUID().get());
keyword("owner_uuid").build(g -> g.getOwnerGroupUUID().get());
/** Timestamp indicating when this group was created. */
public static final FieldDef<InternalGroup, Timestamp> CREATED_ON =

View File

@ -24,13 +24,7 @@ import org.testcontainers.containers.GenericContainer;
public class ElasticContainer<SELF extends ElasticContainer<SELF>> extends GenericContainer<SELF> {
private static final int ELASTICSEARCH_DEFAULT_PORT = 9200;
public enum Version {
V2,
V5,
V6
}
public static ElasticContainer<?> createAndStart(Version version) {
public static ElasticContainer<?> createAndStart(ElasticVersion version) {
// Assumption violation is not natively supported by Testcontainers.
// See https://github.com/testcontainers/testcontainers-java/issues/343
try {
@ -43,22 +37,22 @@ public class ElasticContainer<SELF extends ElasticContainer<SELF>> extends Gener
}
public static ElasticContainer<?> createAndStart() {
return createAndStart(Version.V2);
return createAndStart(ElasticVersion.V2_4);
}
private static String getImageName(Version version) {
private static String getImageName(ElasticVersion version) {
switch (version) {
case V2:
case V2_4:
return "elasticsearch:2.4.6-alpine";
case V5:
case V5_6:
return "elasticsearch:5.6.9-alpine";
case V6:
case V6_2:
return "docker.elastic.co/elasticsearch/elasticsearch:6.2.4";
}
throw new IllegalStateException("Unsupported version: " + version.name());
throw new IllegalStateException("No tests for version: " + version.name());
}
private ElasticContainer(Version version) {
private ElasticContainer(ElasticVersion version) {
super(getImageName(version));
}

View File

@ -17,7 +17,7 @@
set -eu
# Keep this version in sync with dev-contributing.txt.
VERSION=${1:-1.5}
VERSION=${1:-1.6}
case "$VERSION" in
1.3)
@ -26,6 +26,9 @@ case "$VERSION" in
1.5)
SHA1="b1f79e4d39a3c501f07c0ce7e8b03ac6964ed1f1"
;;
1.6)
SHA1="02b3e84e52d2473e2c4868189709905a51647d03"
;;
*)
echo "unknown google-java-format version: $VERSION"
exit 1