diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java index 566e1596bb..65065f9e93 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java @@ -62,7 +62,11 @@ public class ElasticIndexModule extends LifecycleModule { .build(GroupIndex.Factory.class)); install(new IndexModule(threads)); - install(new SingleVersionModule(singleVersions)); + if (singleVersions == null) { + listener().to(ElasticVersionManager.class); + } else { + install(new SingleVersionModule(singleVersions)); + } } @Provides diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java new file mode 100644 index 0000000000..b73b37f556 --- /dev/null +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java @@ -0,0 +1,52 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.searchbox.client.JestResult; +import io.searchbox.client.http.JestHttpClient; +import io.searchbox.indices.aliases.GetAliases; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; + +@Singleton +class ElasticIndexVersionDiscovery { + private final JestHttpClient client; + + @Inject + ElasticIndexVersionDiscovery(JestClientBuilder clientBuilder) { + this.client = clientBuilder.build(); + } + + List discover(String prefix, String indexName) throws IOException { + String name = prefix + indexName + "_"; + JestResult result = client.execute(new GetAliases.Builder().addIndex(name + "*").build()); + if (result.isSucceeded()) { + JsonObject object = result.getJsonObject().getAsJsonObject(); + List versions = new ArrayList<>(object.size()); + for (Entry entry : object.entrySet()) { + versions.add(entry.getKey().replace(name, "")); + } + return versions; + } + return Collections.emptyList(); + } +} diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java new file mode 100644 index 0000000000..917217a0ea --- /dev/null +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java @@ -0,0 +1,251 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.primitives.Ints; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.Index; +import com.google.gerrit.server.index.IndexCollection; +import com.google.gerrit.server.index.IndexDefinition; +import com.google.gerrit.server.index.IndexDefinition.IndexFactory; +import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.OnlineReindexer; +import com.google.gerrit.server.index.ReindexerAlreadyRunningException; +import com.google.gerrit.server.index.Schema; +import com.google.inject.Inject; +import com.google.inject.ProvisionException; +import com.google.inject.Singleton; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.eclipse.jgit.lib.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class ElasticVersionManager implements LifecycleListener { + private static final Logger log = LoggerFactory.getLogger(ElasticVersionManager.class); + + private static class Version { + private final Schema schema; + private final int version; + private final boolean ready; + + private Version(Schema schema, int version, boolean ready) { + checkArgument(schema == null || schema.getVersion() == version); + this.schema = schema; + this.version = version; + this.ready = ready; + } + } + + private final Map> defs; + private final Map> reindexers; + private final ElasticIndexVersionDiscovery versionDiscovery; + private final SitePaths sitePaths; + private final boolean onlineUpgrade; + private final String runReindexMsg; + private final String prefix; + + @Inject + ElasticVersionManager( + @GerritServerConfig Config cfg, + SitePaths sitePaths, + Collection> defs, + ElasticIndexVersionDiscovery versionDiscovery) { + this.sitePaths = sitePaths; + this.versionDiscovery = versionDiscovery; + this.defs = Maps.newHashMapWithExpectedSize(defs.size()); + for (IndexDefinition def : defs) { + this.defs.put(def.getName(), def); + } + + prefix = MoreObjects.firstNonNull(cfg.getString("index", null, "prefix"), "gerrit"); + reindexers = Maps.newHashMapWithExpectedSize(defs.size()); + onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true); + runReindexMsg = + "No index versions ready; run java -jar " + + sitePaths.gerrit_war.toAbsolutePath() + + " reindex"; + } + + @Override + public void start() { + try { + for (IndexDefinition def : defs.values()) { + initIndex(def); + } + } catch (IOException e) { + ProvisionException ex = new ProvisionException("Error scanning indexes"); + ex.initCause(e); + throw ex; + } + } + + private > void initIndex(IndexDefinition def) + throws IOException { + TreeMap> versions = scanVersions(def); + // Search from the most recent ready version. + // Write to the most recent ready version and the most recent version. + Version search = null; + List> write = Lists.newArrayListWithCapacity(2); + for (Version v : versions.descendingMap().values()) { + if (v.schema == null) { + continue; + } + if (write.isEmpty() && onlineUpgrade) { + write.add(v); + } + if (v.ready) { + search = v; + if (!write.contains(v)) { + write.add(v); + } + break; + } + } + if (search == null) { + throw new ProvisionException(runReindexMsg); + } + + IndexFactory factory = def.getIndexFactory(); + I searchIndex = factory.create(search.schema); + IndexCollection indexes = def.getIndexCollection(); + indexes.setSearchIndex(searchIndex); + for (Version v : write) { + if (v.schema != null) { + if (v.version != search.version) { + indexes.addWriteIndex(factory.create(v.schema)); + } else { + indexes.addWriteIndex(searchIndex); + } + } + } + + markNotReady(def.getName(), versions.values(), write); + + synchronized (this) { + if (!reindexers.containsKey(def.getName())) { + int latest = write.get(0).version; + OnlineReindexer reindexer = new OnlineReindexer<>(def, latest); + reindexers.put(def.getName(), reindexer); + if (onlineUpgrade && latest != search.version) { + reindexer.start(); + } + } + } + } + + /** + * Start the online reindexer if the current index is not already the latest. + * + * @param name index name + * @param force start re-index + * @return true if started, otherwise false. + * @throws ReindexerAlreadyRunningException + */ + public synchronized boolean startReindexer(String name, boolean force) + throws ReindexerAlreadyRunningException { + OnlineReindexer reindexer = reindexers.get(name); + validateReindexerNotRunning(reindexer); + if (force || !isLatestIndexVersion(name, reindexer)) { + reindexer.start(); + return true; + } + return false; + } + + /** + * Activate the latest index if the current index is not already the latest. + * + * @param name index name + * @return true if index was activated, otherwise false. + * @throws ReindexerAlreadyRunningException + */ + public synchronized boolean activateLatestIndex(String name) + throws ReindexerAlreadyRunningException { + OnlineReindexer reindexer = reindexers.get(name); + validateReindexerNotRunning(reindexer); + if (!isLatestIndexVersion(name, reindexer)) { + reindexer.activateIndex(); + return true; + } + return false; + } + + private boolean isLatestIndexVersion(String name, OnlineReindexer reindexer) { + int readVersion = defs.get(name).getIndexCollection().getSearchIndex().getSchema().getVersion(); + return reindexer == null || reindexer.getVersion() == readVersion; + } + + private static void validateReindexerNotRunning(OnlineReindexer reindexer) + throws ReindexerAlreadyRunningException { + if (reindexer != null && reindexer.isRunning()) { + throw new ReindexerAlreadyRunningException(); + } + } + + private > TreeMap> scanVersions( + IndexDefinition def) throws IOException { + TreeMap> versions = new TreeMap<>(); + for (Schema schema : def.getSchemas().values()) { + int v = schema.getVersion(); + versions.put( + v, + new Version<>( + schema, v, IndexUtils.getReady(sitePaths, def.getName(), schema.getVersion()))); + } + + try { + for (String version : versionDiscovery.discover(prefix, def.getName())) { + Integer v = Ints.tryParse(version); + if (v == null || version.length() != 4) { + log.warn("Unrecognized version in index {}: {}", def.getName(), version); + continue; + } + if (!versions.containsKey(v)) { + versions.put( + v, new Version(null, v, IndexUtils.getReady(sitePaths, def.getName(), v))); + } + } + } catch (IOException e) { + log.error("Error scanning index: " + def.getName(), e); + } + return versions; + } + + private void markNotReady( + String name, Iterable> versions, Collection> inUse) throws IOException { + for (Version v : versions) { + if (!inUse.contains(v)) { + IndexUtils.getReady(sitePaths, name, v.version); + } + } + } + + @Override + public void stop() { + // Do nothing; indexes are closed on demand by IndexCollection. + } +} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java index 94b3618e82..a2d4f1eaa1 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java @@ -28,6 +28,7 @@ import com.google.gerrit.server.index.IndexCollection; import com.google.gerrit.server.index.IndexDefinition; import com.google.gerrit.server.index.IndexDefinition.IndexFactory; import com.google.gerrit.server.index.OnlineReindexer; +import com.google.gerrit.server.index.ReindexerAlreadyRunningException; import com.google.gerrit.server.index.Schema; import com.google.inject.Inject; import com.google.inject.ProvisionException; diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java index 708e0103e3..7000e04916 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java @@ -43,6 +43,15 @@ public final class IndexUtils { } } + public static boolean getReady(SitePaths sitePaths, String name, int version) throws IOException { + try { + GerritIndexStatus cfg = new GerritIndexStatus(sitePaths); + return cfg.getReady(name, version); + } catch (ConfigInvalidException e) { + throw new IOException(e); + } + } + public static Set accountFields(QueryOptions opts) { Set fs = opts.fields(); return fs.contains(AccountField.ID.getName()) diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/ReindexerAlreadyRunningException.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ReindexerAlreadyRunningException.java similarity index 95% rename from gerrit-lucene/src/main/java/com/google/gerrit/lucene/ReindexerAlreadyRunningException.java rename to gerrit-server/src/main/java/com/google/gerrit/server/index/ReindexerAlreadyRunningException.java index 0ca632bc71..8bf99a5165 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/ReindexerAlreadyRunningException.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ReindexerAlreadyRunningException.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.google.gerrit.lucene; +package com.google.gerrit.server.index; public class ReindexerAlreadyRunningException extends Exception { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/SingleVersionModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/SingleVersionModule.java index 2df0b40089..bf28d7d31b 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/SingleVersionModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/SingleVersionModule.java @@ -24,6 +24,7 @@ import com.google.inject.Singleton; import com.google.inject.TypeLiteral; import com.google.inject.name.Named; import com.google.inject.name.Names; +import com.google.inject.util.Providers; import java.util.Collection; import java.util.Map; import java.util.Set; @@ -44,7 +45,7 @@ public class SingleVersionModule extends LifecycleModule { listener().to(SingleVersionListener.class); bind(new TypeLiteral>() {}) .annotatedWith(Names.named(SINGLE_VERSIONS)) - .toInstance(singleVersions); + .toProvider(Providers.of(singleVersions)); } @Singleton diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexActivateCommand.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexActivateCommand.java index bff14dd05d..6b3e6d7828 100644 --- a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexActivateCommand.java +++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexActivateCommand.java @@ -17,7 +17,7 @@ package com.google.gerrit.sshd.commands; import com.google.gerrit.common.data.GlobalCapability; import com.google.gerrit.extensions.annotations.RequiresCapability; import com.google.gerrit.lucene.LuceneVersionManager; -import com.google.gerrit.lucene.ReindexerAlreadyRunningException; +import com.google.gerrit.server.index.ReindexerAlreadyRunningException; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; import com.google.inject.Inject; diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexStartCommand.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexStartCommand.java index cee016c356..fb9b4829a5 100644 --- a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexStartCommand.java +++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/IndexStartCommand.java @@ -17,7 +17,7 @@ package com.google.gerrit.sshd.commands; import com.google.gerrit.common.data.GlobalCapability; import com.google.gerrit.extensions.annotations.RequiresCapability; import com.google.gerrit.lucene.LuceneVersionManager; -import com.google.gerrit.lucene.ReindexerAlreadyRunningException; +import com.google.gerrit.server.index.ReindexerAlreadyRunningException; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; import com.google.inject.Inject;