diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java index 917217a0ea..74a6b69c0d 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticVersionManager.java @@ -14,59 +14,31 @@ package com.google.gerrit.elasticsearch; -import static com.google.common.base.Preconditions.checkArgument; - import com.google.common.base.MoreObjects; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AbstractVersionManager; +import com.google.gerrit.server.index.GerritIndexStatus; import com.google.gerrit.server.index.Index; -import com.google.gerrit.server.index.IndexCollection; import com.google.gerrit.server.index.IndexDefinition; -import com.google.gerrit.server.index.IndexDefinition.IndexFactory; -import com.google.gerrit.server.index.IndexUtils; -import com.google.gerrit.server.index.OnlineReindexer; -import com.google.gerrit.server.index.ReindexerAlreadyRunningException; import com.google.gerrit.server.index.Schema; import com.google.inject.Inject; -import com.google.inject.ProvisionException; import com.google.inject.Singleton; import java.io.IOException; import java.util.Collection; -import java.util.List; -import java.util.Map; import java.util.TreeMap; import org.eclipse.jgit.lib.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton -public class ElasticVersionManager implements LifecycleListener { +public class ElasticVersionManager extends AbstractVersionManager implements LifecycleListener { private static final Logger log = LoggerFactory.getLogger(ElasticVersionManager.class); - private static class Version { - private final Schema schema; - private final int version; - private final boolean ready; - - private Version(Schema schema, int version, boolean ready) { - checkArgument(schema == null || schema.getVersion() == version); - this.schema = schema; - this.version = version; - this.ready = ready; - } - } - - private final Map> defs; - private final Map> reindexers; - private final ElasticIndexVersionDiscovery versionDiscovery; - private final SitePaths sitePaths; - private final boolean onlineUpgrade; - private final String runReindexMsg; private final String prefix; + private final ElasticIndexVersionDiscovery versionDiscovery; @Inject ElasticVersionManager( @@ -74,147 +46,23 @@ public class ElasticVersionManager implements LifecycleListener { SitePaths sitePaths, Collection> defs, ElasticIndexVersionDiscovery versionDiscovery) { - this.sitePaths = sitePaths; + super(cfg, sitePaths, defs); this.versionDiscovery = versionDiscovery; - this.defs = Maps.newHashMapWithExpectedSize(defs.size()); - for (IndexDefinition def : defs) { - this.defs.put(def.getName(), def); - } - prefix = MoreObjects.firstNonNull(cfg.getString("index", null, "prefix"), "gerrit"); - reindexers = Maps.newHashMapWithExpectedSize(defs.size()); - onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true); - runReindexMsg = - "No index versions ready; run java -jar " - + sitePaths.gerrit_war.toAbsolutePath() - + " reindex"; } @Override - public void start() { - try { - for (IndexDefinition def : defs.values()) { - initIndex(def); - } - } catch (IOException e) { - ProvisionException ex = new ProvisionException("Error scanning indexes"); - ex.initCause(e); - throw ex; - } + protected boolean isDirty(Collection> inUse, Version v) { + return !inUse.contains(v); } - private > void initIndex(IndexDefinition def) - throws IOException { - TreeMap> versions = scanVersions(def); - // Search from the most recent ready version. - // Write to the most recent ready version and the most recent version. - Version search = null; - List> write = Lists.newArrayListWithCapacity(2); - for (Version v : versions.descendingMap().values()) { - if (v.schema == null) { - continue; - } - if (write.isEmpty() && onlineUpgrade) { - write.add(v); - } - if (v.ready) { - search = v; - if (!write.contains(v)) { - write.add(v); - } - break; - } - } - if (search == null) { - throw new ProvisionException(runReindexMsg); - } - - IndexFactory factory = def.getIndexFactory(); - I searchIndex = factory.create(search.schema); - IndexCollection indexes = def.getIndexCollection(); - indexes.setSearchIndex(searchIndex); - for (Version v : write) { - if (v.schema != null) { - if (v.version != search.version) { - indexes.addWriteIndex(factory.create(v.schema)); - } else { - indexes.addWriteIndex(searchIndex); - } - } - } - - markNotReady(def.getName(), versions.values(), write); - - synchronized (this) { - if (!reindexers.containsKey(def.getName())) { - int latest = write.get(0).version; - OnlineReindexer reindexer = new OnlineReindexer<>(def, latest); - reindexers.put(def.getName(), reindexer); - if (onlineUpgrade && latest != search.version) { - reindexer.start(); - } - } - } - } - - /** - * Start the online reindexer if the current index is not already the latest. - * - * @param name index name - * @param force start re-index - * @return true if started, otherwise false. - * @throws ReindexerAlreadyRunningException - */ - public synchronized boolean startReindexer(String name, boolean force) - throws ReindexerAlreadyRunningException { - OnlineReindexer reindexer = reindexers.get(name); - validateReindexerNotRunning(reindexer); - if (force || !isLatestIndexVersion(name, reindexer)) { - reindexer.start(); - return true; - } - return false; - } - - /** - * Activate the latest index if the current index is not already the latest. - * - * @param name index name - * @return true if index was activated, otherwise false. - * @throws ReindexerAlreadyRunningException - */ - public synchronized boolean activateLatestIndex(String name) - throws ReindexerAlreadyRunningException { - OnlineReindexer reindexer = reindexers.get(name); - validateReindexerNotRunning(reindexer); - if (!isLatestIndexVersion(name, reindexer)) { - reindexer.activateIndex(); - return true; - } - return false; - } - - private boolean isLatestIndexVersion(String name, OnlineReindexer reindexer) { - int readVersion = defs.get(name).getIndexCollection().getSearchIndex().getSchema().getVersion(); - return reindexer == null || reindexer.getVersion() == readVersion; - } - - private static void validateReindexerNotRunning(OnlineReindexer reindexer) - throws ReindexerAlreadyRunningException { - if (reindexer != null && reindexer.isRunning()) { - throw new ReindexerAlreadyRunningException(); - } - } - - private > TreeMap> scanVersions( - IndexDefinition def) throws IOException { + @Override + protected > TreeMap> scanVersions( + IndexDefinition def, GerritIndexStatus cfg) { TreeMap> versions = new TreeMap<>(); for (Schema schema : def.getSchemas().values()) { int v = schema.getVersion(); - versions.put( - v, - new Version<>( - schema, v, IndexUtils.getReady(sitePaths, def.getName(), schema.getVersion()))); + versions.put(v, new Version<>(schema, v, cfg.getReady(def.getName(), v))); } try { @@ -225,8 +73,7 @@ public class ElasticVersionManager implements LifecycleListener { continue; } if (!versions.containsKey(v)) { - versions.put( - v, new Version(null, v, IndexUtils.getReady(sitePaths, def.getName(), v))); + versions.put(v, new Version(null, v, cfg.getReady(def.getName(), v))); } } } catch (IOException e) { @@ -234,18 +81,4 @@ public class ElasticVersionManager implements LifecycleListener { } return versions; } - - private void markNotReady( - String name, Iterable> versions, Collection> inUse) throws IOException { - for (Version v : versions) { - if (!inUse.contains(v)) { - IndexUtils.getReady(sitePaths, name, v.version); - } - } - } - - @Override - public void stop() { - // Do nothing; indexes are closed on demand by IndexCollection. - } } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java index a2d4f1eaa1..f6f0c287ff 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java @@ -14,54 +14,37 @@ package com.google.gerrit.lucene; -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AbstractVersionManager; import com.google.gerrit.server.index.GerritIndexStatus; import com.google.gerrit.server.index.Index; -import com.google.gerrit.server.index.IndexCollection; import com.google.gerrit.server.index.IndexDefinition; -import com.google.gerrit.server.index.IndexDefinition.IndexFactory; -import com.google.gerrit.server.index.OnlineReindexer; -import com.google.gerrit.server.index.ReindexerAlreadyRunningException; import com.google.gerrit.server.index.Schema; import com.google.inject.Inject; -import com.google.inject.ProvisionException; import com.google.inject.Singleton; import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; -import java.util.List; -import java.util.Map; import java.util.TreeMap; -import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton -public class LuceneVersionManager implements LifecycleListener { +public class LuceneVersionManager extends AbstractVersionManager implements LifecycleListener { private static final Logger log = LoggerFactory.getLogger(LuceneVersionManager.class); - private static class Version { - private final Schema schema; - private final int version; + private static class Version extends AbstractVersionManager.Version { private final boolean exists; - private final boolean ready; private Version(Schema schema, int version, boolean exists, boolean ready) { - checkArgument(schema == null || schema.getVersion() == version); - this.schema = schema; - this.version = version; + super(schema, version, ready); this.exists = exists; - this.ready = ready; } } @@ -69,156 +52,26 @@ public class LuceneVersionManager implements LifecycleListener { return sitePaths.index_dir.resolve(String.format("%s%04d", prefix, schema.getVersion())); } - private final SitePaths sitePaths; - private final Map> defs; - private final Map> reindexers; - private final boolean onlineUpgrade; - private final String runReindexMsg; - @Inject LuceneVersionManager( @GerritServerConfig Config cfg, SitePaths sitePaths, Collection> defs) { - this.sitePaths = sitePaths; - this.defs = Maps.newHashMapWithExpectedSize(defs.size()); - for (IndexDefinition def : defs) { - this.defs.put(def.getName(), def); - } - - reindexers = Maps.newHashMapWithExpectedSize(defs.size()); - onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true); - runReindexMsg = - "No index versions ready; run java -jar " - + sitePaths.gerrit_war.toAbsolutePath() - + " reindex"; + super(cfg, sitePaths, defs); } @Override - public void start() { - GerritIndexStatus cfg; - try { - cfg = new GerritIndexStatus(sitePaths); - } catch (ConfigInvalidException | IOException e) { - throw fail(e); - } - - if (!Files.exists(sitePaths.index_dir)) { - throw new ProvisionException(runReindexMsg); - } else if (!Files.exists(sitePaths.index_dir)) { - log.warn("Not a directory: %s", sitePaths.index_dir.toAbsolutePath()); - throw new ProvisionException(runReindexMsg); - } - - for (IndexDefinition def : defs.values()) { - initIndex(def, cfg); - } + protected boolean isDirty( + Collection> inUse, + com.google.gerrit.server.index.AbstractVersionManager.Version v) { + return !inUse.contains(v) && ((Version) v).exists; } - private > void initIndex( - IndexDefinition def, GerritIndexStatus cfg) { - TreeMap> versions = scanVersions(def, cfg); - // Search from the most recent ready version. - // Write to the most recent ready version and the most recent version. - Version search = null; - List> write = Lists.newArrayListWithCapacity(2); - for (Version v : versions.descendingMap().values()) { - if (v.schema == null) { - continue; - } - if (write.isEmpty() && onlineUpgrade) { - write.add(v); - } - if (v.ready) { - search = v; - if (!write.contains(v)) { - write.add(v); - } - break; - } - } - if (search == null) { - throw new ProvisionException(runReindexMsg); - } - - IndexFactory factory = def.getIndexFactory(); - I searchIndex = factory.create(search.schema); - IndexCollection indexes = def.getIndexCollection(); - indexes.setSearchIndex(searchIndex); - for (Version v : write) { - if (v.schema != null) { - if (v.version != search.version) { - indexes.addWriteIndex(factory.create(v.schema)); - } else { - indexes.addWriteIndex(searchIndex); - } - } - } - - markNotReady(cfg, def.getName(), versions.values(), write); - - int latest = write.get(0).version; - OnlineReindexer reindexer = new OnlineReindexer<>(def, latest); - synchronized (this) { - if (!reindexers.containsKey(def.getName())) { - reindexers.put(def.getName(), reindexer); - if (onlineUpgrade && latest != search.version) { - reindexer.start(); - } - } - } - } - - /** - * Start the online reindexer if the current index is not already the latest. - * - * @param force start re-index - * @return true if started, otherwise false. - * @throws ReindexerAlreadyRunningException - */ - public synchronized boolean startReindexer(String name, boolean force) - throws ReindexerAlreadyRunningException { - OnlineReindexer reindexer = reindexers.get(name); - validateReindexerNotRunning(reindexer); - if (force || !isCurrentIndexVersionLatest(name, reindexer)) { - reindexer.start(); - return true; - } - return false; - } - - /** - * Activate the latest index if the current index is not already the latest. - * - * @return true if index was activate, otherwise false. - * @throws ReindexerAlreadyRunningException - */ - public synchronized boolean activateLatestIndex(String name) - throws ReindexerAlreadyRunningException { - OnlineReindexer reindexer = reindexers.get(name); - validateReindexerNotRunning(reindexer); - if (!isCurrentIndexVersionLatest(name, reindexer)) { - reindexer.activateIndex(); - return true; - } - return false; - } - - private boolean isCurrentIndexVersionLatest(String name, OnlineReindexer reindexer) { - int readVersion = defs.get(name).getIndexCollection().getSearchIndex().getSchema().getVersion(); - return reindexer == null || reindexer.getVersion() == readVersion; - } - - private static void validateReindexerNotRunning(OnlineReindexer reindexer) - throws ReindexerAlreadyRunningException { - if (reindexer != null && reindexer.isRunning()) { - throw new ReindexerAlreadyRunningException(); - } - } - - private > TreeMap> scanVersions( - IndexDefinition def, GerritIndexStatus cfg) { - TreeMap> versions = new TreeMap<>(); + @Override + protected > + TreeMap> scanVersions( + IndexDefinition def, GerritIndexStatus cfg) { + TreeMap> versions = new TreeMap<>(); for (Schema schema : def.getSchemas().values()) { // This part is Lucene-specific. Path p = getDir(sitePaths, def.getName(), schema); @@ -252,36 +105,4 @@ public class LuceneVersionManager implements LifecycleListener { } return versions; } - - private void markNotReady( - GerritIndexStatus cfg, - String name, - Iterable> versions, - Collection> inUse) { - boolean dirty = false; - for (Version v : versions) { - if (!inUse.contains(v) && v.exists) { - cfg.setReady(name, v.version, false); - dirty = true; - } - } - if (dirty) { - try { - cfg.save(); - } catch (IOException e) { - throw fail(e); - } - } - } - - private ProvisionException fail(Throwable t) { - ProvisionException e = new ProvisionException("Error scanning indexes"); - e.initCause(t); - throw e; - } - - @Override - public void stop() { - // Do nothing; indexes are closed on demand by IndexCollection. - } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/AbstractVersionManager.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/AbstractVersionManager.java new file mode 100644 index 0000000000..33cca1e160 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/AbstractVersionManager.java @@ -0,0 +1,223 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexDefinition.IndexFactory; +import com.google.inject.ProvisionException; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; + +public abstract class AbstractVersionManager implements LifecycleListener { + public static class Version { + public final Schema schema; + public final int version; + public final boolean ready; + + public Version(Schema schema, int version, boolean ready) { + checkArgument(schema == null || schema.getVersion() == version); + this.schema = schema; + this.version = version; + this.ready = ready; + } + } + + protected final boolean onlineUpgrade; + protected final String runReindexMsg; + protected final SitePaths sitePaths; + protected final Map> defs; + protected final Map> reindexers; + + protected AbstractVersionManager( + @GerritServerConfig Config cfg, + SitePaths sitePaths, + Collection> defs) { + this.sitePaths = sitePaths; + this.defs = Maps.newHashMapWithExpectedSize(defs.size()); + for (IndexDefinition def : defs) { + this.defs.put(def.getName(), def); + } + + reindexers = Maps.newHashMapWithExpectedSize(defs.size()); + onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true); + runReindexMsg = + "No index versions ready; run java -jar " + + sitePaths.gerrit_war.toAbsolutePath() + + " reindex"; + } + + @Override + public void start() { + GerritIndexStatus cfg = createIndexStatus(); + for (IndexDefinition def : defs.values()) { + initIndex(def, cfg); + } + } + + @Override + public void stop() { + // Do nothing; indexes are closed on demand by IndexCollection. + } + + /** + * Start the online reindexer if the current index is not already the latest. + * + * @param name index name + * @param force start re-index + * @return true if started, otherwise false. + * @throws ReindexerAlreadyRunningException + */ + public synchronized boolean startReindexer(String name, boolean force) + throws ReindexerAlreadyRunningException { + OnlineReindexer reindexer = reindexers.get(name); + validateReindexerNotRunning(reindexer); + if (force || !isLatestIndexVersion(name, reindexer)) { + reindexer.start(); + return true; + } + return false; + } + + /** + * Activate the latest index if the current index is not already the latest. + * + * @param name index name + * @return true if index was activated, otherwise false. + * @throws ReindexerAlreadyRunningException + */ + public synchronized boolean activateLatestIndex(String name) + throws ReindexerAlreadyRunningException { + OnlineReindexer reindexer = reindexers.get(name); + validateReindexerNotRunning(reindexer); + if (!isLatestIndexVersion(name, reindexer)) { + reindexer.activateIndex(); + return true; + } + return false; + } + + protected > void initIndex( + IndexDefinition def, GerritIndexStatus cfg) { + TreeMap> versions = scanVersions(def, cfg); + // Search from the most recent ready version. + // Write to the most recent ready version and the most recent version. + Version search = null; + List> write = Lists.newArrayListWithCapacity(2); + for (Version v : versions.descendingMap().values()) { + if (v.schema == null) { + continue; + } + if (write.isEmpty() && onlineUpgrade) { + write.add(v); + } + if (v.ready) { + search = v; + if (!write.contains(v)) { + write.add(v); + } + break; + } + } + if (search == null) { + throw new ProvisionException(runReindexMsg); + } + + IndexFactory factory = def.getIndexFactory(); + I searchIndex = factory.create(search.schema); + IndexCollection indexes = def.getIndexCollection(); + indexes.setSearchIndex(searchIndex); + for (Version v : write) { + if (v.version != search.version) { + indexes.addWriteIndex(factory.create(v.schema)); + } else { + indexes.addWriteIndex(searchIndex); + } + } + + markNotReady(def.getName(), versions.values(), write); + + synchronized (this) { + if (!reindexers.containsKey(def.getName())) { + int latest = write.get(0).version; + OnlineReindexer reindexer = new OnlineReindexer<>(def, latest); + reindexers.put(def.getName(), reindexer); + if (onlineUpgrade && latest != search.version) { + reindexer.start(); + } + } + } + } + + protected GerritIndexStatus createIndexStatus() { + try { + return new GerritIndexStatus(sitePaths); + } catch (ConfigInvalidException | IOException e) { + throw fail(e); + } + } + + protected abstract boolean isDirty(Collection> inUse, Version v); + + protected abstract > TreeMap> scanVersions( + IndexDefinition def, GerritIndexStatus cfg); + + private boolean isLatestIndexVersion(String name, OnlineReindexer reindexer) { + int readVersion = defs.get(name).getIndexCollection().getSearchIndex().getSchema().getVersion(); + return reindexer == null || reindexer.getVersion() == readVersion; + } + + private static void validateReindexerNotRunning(OnlineReindexer reindexer) + throws ReindexerAlreadyRunningException { + if (reindexer != null && reindexer.isRunning()) { + throw new ReindexerAlreadyRunningException(); + } + } + + private void markNotReady( + String name, Iterable> versions, Collection> inUse) { + GerritIndexStatus cfg = createIndexStatus(); + boolean dirty = false; + for (Version v : versions) { + if (isDirty(inUse, v)) { + cfg.setReady(name, v.version, false); + dirty = true; + } + } + if (dirty) { + try { + cfg.save(); + } catch (IOException e) { + throw fail(e); + } + } + } + + private ProvisionException fail(Throwable t) { + ProvisionException e = new ProvisionException("Error scanning indexes"); + e.initCause(t); + return e; + } +}