diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK index 5e9f376e75..b5e5d07f69 100644 --- a/gerrit-lucene/BUCK +++ b/gerrit-lucene/BUCK @@ -27,7 +27,9 @@ java_library( '//gerrit-server:server', '//lib:guava', '//lib:gwtorm', + '//lib:jsr305', '//lib/guice:guice', + '//lib/guice:guice-assistedinject', '//lib/jgit:jgit', '//lib/log:api', '//lib/lucene:analyzers-common', diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java deleted file mode 100644 index 54591c997d..0000000000 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (C) 2013 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.lucene; - -import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION; -import static org.apache.lucene.util.Version.LUCENE_CURRENT; - -import com.google.common.collect.ImmutableMap; -import com.google.gerrit.extensions.events.LifecycleListener; -import com.google.gerrit.server.config.SitePaths; -import com.google.gerrit.server.index.ChangeSchemas; -import com.google.inject.Inject; -import com.google.inject.ProvisionException; - -import org.apache.lucene.util.Version; -import org.eclipse.jgit.errors.ConfigInvalidException; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.util.FS; - -import java.io.File; -import java.io.IOException; -import java.util.Map; - -public class IndexVersionCheck implements LifecycleListener { - public static final Map SCHEMA_VERSIONS = ImmutableMap.of( - LuceneChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(), - LuceneChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion()); - - public static File gerritIndexConfig(SitePaths sitePaths) { - return new File(sitePaths.index_dir, "gerrit_index.config"); - } - - private final SitePaths sitePaths; - - @Inject - IndexVersionCheck(SitePaths sitePaths) { - this.sitePaths = sitePaths; - } - - @Override - public void start() { - File file = gerritIndexConfig(sitePaths); - try { - FileBasedConfig cfg = new FileBasedConfig(file, FS.detect()); - cfg.load(); - for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { - int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0); - if (schemaVersion != e.getValue()) { - throw new ProvisionException(String.format( - "wrong index schema version for \"%s\": expected %d, found %d%s", - e.getKey(), e.getValue(), schemaVersion, upgrade())); - } - } - @SuppressWarnings("deprecation") - Version luceneVersion = - cfg.getEnum("lucene", null, "version", LUCENE_CURRENT); - if (luceneVersion != LUCENE_VERSION) { - throw new ProvisionException(String.format( - "wrong Lucene version: expected %d, found %d%s", - luceneVersion, LUCENE_VERSION, upgrade())); - - } - } catch (IOException e) { - throw new ProvisionException("unable to read " + file); - } catch (ConfigInvalidException e) { - throw new ProvisionException("invalid config file " + file); - } - } - - @Override - public void stop() { - // Do nothing. - } - - private final String upgrade() { - return "\nRun reindex to rebuild the index:\n" - + "$ java -jar gerrit.war reindex -d " - + sitePaths.site_path.getAbsolutePath(); - } -} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index ef2ef91784..f87b328add 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -14,8 +14,6 @@ package com.google.gerrit.lucene; -import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS; -import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig; import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES; import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES; @@ -26,7 +24,6 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; @@ -35,7 +32,6 @@ import com.google.gerrit.server.index.ChangeIndex; import com.google.gerrit.server.index.FieldDef; import com.google.gerrit.server.index.FieldDef.FillArgs; import com.google.gerrit.server.index.FieldType; -import com.google.gerrit.server.index.IndexCollection; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.IndexRewriteImpl; import com.google.gerrit.server.index.Schema; @@ -45,6 +41,8 @@ import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.ChangeDataSource; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.ResultSet; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; @@ -69,7 +67,6 @@ import org.apache.lucene.util.Version; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.util.FS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,11 +76,12 @@ import java.sql.Timestamp; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import javax.annotation.Nullable; + /** * Secondary index implementation using Apache Lucene. *

@@ -92,7 +90,7 @@ import java.util.concurrent.Future; * though there may be some lag between a committed write and it showing up to * other threads' searchers. */ -public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { +public class LuceneChangeIndex implements ChangeIndex { private static final Logger log = LoggerFactory.getLogger(LuceneChangeIndex.class); @@ -101,6 +99,10 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { public static final String CHANGES_CLOSED = "closed"; private static final String ID_FIELD = ChangeField.LEGACY_ID.getName(); + static interface Factory { + LuceneChangeIndex create(Schema schema, String base); + } + private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) { IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)); @@ -115,28 +117,27 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { private final SitePaths sitePaths; private final FillArgs fillArgs; - private final IndexCollection indexes; private final ExecutorService executor; + private final File dir; private final Schema schema; private final SubIndex openIndex; private final SubIndex closedIndex; - LuceneChangeIndex(@GerritServerConfig Config cfg, + @AssistedInject + LuceneChangeIndex( + @GerritServerConfig Config cfg, SitePaths sitePaths, - IndexCollection indexes, @IndexExecutor ListeningScheduledExecutorService executor, FillArgs fillArgs, - Schema schema, - String base) throws IOException { - this.indexes = indexes; + @Assisted Schema schema, + @Assisted @Nullable String base) throws IOException { this.sitePaths = sitePaths; this.fillArgs = fillArgs; this.executor = executor; this.schema = schema; - File dir; if (base == null) { - dir = new File(sitePaths.index_dir, "changes_" + schema.getVersion()); + dir = LuceneVersionManager.getDir(sitePaths, schema); } else { dir = new File(base); } @@ -147,13 +148,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { } @Override - public void start() { - indexes.setSearchIndex(this); - indexes.addWriteIndex(this); - } - - @Override - public void stop() { + public void close() { List> closeFutures = Lists.newArrayListWithCapacity(2); closeFutures.add(executor.submit(new Runnable() { @Override @@ -249,6 +244,18 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { return new QuerySource(indexes, QueryBuilder.toQuery(p)); } + @Override + public void markReady() throws IOException { + try { + FileBasedConfig cfg = LuceneVersionManager.loadGerritIndexConfig(sitePaths); + cfg.setBoolean("index", Integer.toString(schema.getVersion()), "ready", + true); + cfg.save(); + } catch (ConfigInvalidException e) { + throw new IOException(e); + } + } + private static class QuerySource implements ChangeDataSource { // TODO(dborowitz): Push limit down from predicate tree. private static final int LIMIT = 1000; @@ -388,17 +395,4 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { private static Field.Store store(FieldDef f) { return f.isStored() ? Field.Store.YES : Field.Store.NO; } - - @Override - public void finishIndex() throws IOException, - ConfigInvalidException { - FileBasedConfig cfg = - new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect()); - - for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { - cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue()); - } - cfg.setEnum("lucene", null, "version", LUCENE_VERSION); - cfg.save(); - } } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java index a302f6fa0f..1e1dca75c3 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -14,57 +14,89 @@ package com.google.gerrit.lucene; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.lifecycle.LifecycleModule; -import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.FactoryModule; import com.google.gerrit.server.config.SitePaths; -import com.google.gerrit.server.index.ChangeIndex; import com.google.gerrit.server.index.ChangeSchemas; -import com.google.gerrit.server.index.FieldDef.FillArgs; import com.google.gerrit.server.index.IndexCollection; -import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.IndexModule; +import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.inject.Inject; import com.google.inject.Provides; import com.google.inject.Singleton; -import org.eclipse.jgit.lib.Config; - -import java.io.IOException; - public class LuceneIndexModule extends LifecycleModule { - private final boolean checkVersion; + private final Integer singleVersion; private final int threads; private final String base; public LuceneIndexModule() { - this(true, 0, null); + this(null, 0, null); } - public LuceneIndexModule(boolean checkVersion, int threads, + public LuceneIndexModule(Integer singleVersion, int threads, String base) { - this.checkVersion = checkVersion; + this.singleVersion = singleVersion; this.threads = threads; this.base = base; } @Override protected void configure() { + install(new FactoryModule() { + @Override + public void configure() { + factory(LuceneChangeIndex.Factory.class); + } + }); install(new IndexModule(threads)); - bind(ChangeIndex.class).to(LuceneChangeIndex.class); - listener().to(LuceneChangeIndex.class); - if (checkVersion) { - listener().to(IndexVersionCheck.class); + if (singleVersion == null && base == null) { + listener().to(LuceneVersionManager.class); + } else { + install(new SingleVersionModule()); + } + } + + private class SingleVersionModule extends LifecycleModule { + @Override + public void configure() { + listener().to(SingleVersionListener.class); + } + + @Provides + @Singleton + LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory, + SitePaths sitePaths) { + Schema schema = singleVersion != null + ? ChangeSchemas.get(singleVersion) + : ChangeSchemas.getLatest(); + return factory.create(schema, base); } } - @Provides @Singleton - public LuceneChangeIndex getChangeIndex(@GerritServerConfig Config cfg, - SitePaths sitePaths, - IndexCollection indexes, - @IndexExecutor ListeningScheduledExecutorService executor, - FillArgs fillArgs) throws IOException { - return new LuceneChangeIndex(cfg, sitePaths, indexes, executor, fillArgs, - ChangeSchemas.getLatestRelease(), base); + static class SingleVersionListener implements LifecycleListener { + private final IndexCollection indexes; + private final LuceneChangeIndex index; + + @Inject + SingleVersionListener(IndexCollection indexes, + LuceneChangeIndex index) { + this.indexes = indexes; + this.index = index; + } + + @Override + public void start() { + indexes.setSearchIndex(index); + indexes.addWriteIndex(index); + } + + @Override + public void stop() { + index.close(); + } } } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java new file mode 100644 index 0000000000..c63eb24c07 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java @@ -0,0 +1,209 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package com.google.gerrit.server.git; + +package com.google.gerrit.lucene; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.primitives.Ints; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.ChangeSchemas; +import com.google.gerrit.server.index.IndexCollection; +import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.inject.Inject; +import com.google.inject.ProvisionException; +import com.google.inject.Singleton; + +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.TreeMap; + +@Singleton +class LuceneVersionManager implements LifecycleListener { + private static final Logger log = LoggerFactory + .getLogger(LuceneVersionManager.class); + + private static final String CHANGES_PREFIX = "changes_"; + + private static class Version { + private final Schema schema; + private final int version; + private final boolean exists; + private final boolean ready; + + private Version(Schema schema, int version, boolean exists, + boolean ready) { + checkArgument(schema == null || schema.getVersion() == version); + this.schema = schema; + this.version = version; + this.exists = exists; + this.ready = ready; + } + } + + static File getDir(SitePaths sitePaths, Schema schema) { + return new File(sitePaths.index_dir, String.format("%s%04d", + CHANGES_PREFIX, schema.getVersion())); + } + + static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths) + throws ConfigInvalidException, IOException { + FileBasedConfig cfg = new FileBasedConfig( + new File(sitePaths.index_dir, "gerrit_index.config"), FS.detect()); + cfg.load(); + return cfg; + } + + private static boolean getReady(Config cfg, int version) { + return cfg.getBoolean("index", Integer.toString(version), "ready", false); + } + + private static void setReady(Config cfg, int version, boolean ready) { + cfg.setBoolean("index", Integer.toString(version), "ready", ready); + } + + private final SitePaths sitePaths; + private final LuceneChangeIndex.Factory indexFactory; + private final IndexCollection indexes; + + @Inject + LuceneVersionManager( + SitePaths sitePaths, + LuceneChangeIndex.Factory indexFactory, + IndexCollection indexes) { + this.sitePaths = sitePaths; + this.indexFactory = indexFactory; + this.indexes = indexes; + } + + @Override + public void start() { + FileBasedConfig cfg; + try { + cfg = loadGerritIndexConfig(sitePaths); + } catch (ConfigInvalidException e) { + throw fail(e); + } catch (IOException e) { + throw fail(e); + } + + TreeMap versions = scanVersions(cfg); + // Search from the most recent ready version. + // Write to the most recent ready version and the most recent version. + Version search = null; + List write = Lists.newArrayListWithCapacity(2); + for (Version v : versions.descendingMap().values()) { + if (v.schema == null) { + continue; + } + if (write.isEmpty()) { + write.add(v); + } + if (v.ready) { + search = v; + if (!write.contains(v)) { + write.add(v); + } + break; + } + } + if (search == null) { + throw new ProvisionException("No index versions ready; run Reindex"); + } + + markNotReady(cfg, versions.values(), write); + LuceneChangeIndex searchIndex = indexFactory.create(search.schema, null); + indexes.setSearchIndex(searchIndex); + for (Version v : write) { + if (v.schema != null) { + if (v.version != search.version) { + indexes.addWriteIndex(indexFactory.create(v.schema, null)); + } else { + indexes.addWriteIndex(searchIndex); + } + } + } + } + + private TreeMap scanVersions(Config cfg) { + TreeMap versions = Maps.newTreeMap(); + for (Schema schema : ChangeSchemas.ALL.values()) { + File f = getDir(sitePaths, schema); + boolean exists = f.exists() && f.isDirectory(); + if (exists && !f.isDirectory()) { + log.warn("Not a directory: %s", f.getAbsolutePath()); + } + int v = schema.getVersion(); + versions.put(v, new Version(schema, v, exists, getReady(cfg, v))); + } + + for (File f : sitePaths.index_dir.listFiles()) { + if (!f.getName().startsWith(CHANGES_PREFIX)) { + continue; + } + String versionStr = f.getName().substring(CHANGES_PREFIX.length()); + Integer v = Ints.tryParse(versionStr); + if (v == null || versionStr.length() != 4) { + log.warn("Unrecognized version in index directory: {}", + f.getAbsolutePath()); + continue; + } + if (!versions.containsKey(v)) { + versions.put(v, new Version(null, v, true, getReady(cfg, v))); + } + } + return versions; + } + + private void markNotReady(FileBasedConfig cfg, Iterable versions, + Collection inUse) { + boolean dirty = false; + for (Version v : versions) { + if (!inUse.contains(v) && v.exists) { + setReady(cfg, v.version, false); + dirty = true; + } + } + if (dirty) { + try { + cfg.save(); + } catch (IOException e) { + throw fail(e); + } + } + } + + private ProvisionException fail(Throwable t) { + ProvisionException e = new ProvisionException("Error scanning indexes"); + e.initCause(t); + throw e; + } + + @Override + public void stop() { + } +} diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java index 1070a52200..6d47124834 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java @@ -43,6 +43,8 @@ import com.google.gerrit.server.git.MultiProgressMonitor; import com.google.gerrit.server.git.MultiProgressMonitor.Task; import com.google.gerrit.server.index.ChangeIndex; import com.google.gerrit.server.index.ChangeIndexer; +import com.google.gerrit.server.index.ChangeSchemas; +import com.google.gerrit.server.index.IndexCollection; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; @@ -63,7 +65,6 @@ import com.google.inject.TypeLiteral; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffFormatter; -import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; @@ -98,14 +99,22 @@ public class Reindex extends SiteProgram { @Option(name = "--threads", usage = "Number of threads to use for indexing") private int threads = Runtime.getRuntime().availableProcessors(); + @Option(name = "--schema-version", + usage = "Schema version to reindex; default is most recent version") + private Integer version; + @Option(name = "--output", usage = "Prefix for output; path for local disk index, or prefix for remote index") private String outputBase; @Option(name = "--verbose", usage = "Output debug information for each change") private boolean verbose; + @Option(name = "--dry-run", usage = "Dry run: don't write anything to index") + private boolean dryRun; + private Injector dbInjector; private Injector sysInjector; + private ChangeIndex index; @Override public int run() throws Exception { @@ -114,22 +123,22 @@ public class Reindex extends SiteProgram { if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) { throw die("index.type must be configured (or not SQL)"); } - + if (version == null) { + version = ChangeSchemas.getLatest().getVersion(); + } LifecycleManager dbManager = new LifecycleManager(); dbManager.add(dbInjector); dbManager.start(); sysInjector = createSysInjector(); - - // Delete before any index may be created depending on this data. - deleteAll(); - LifecycleManager sysManager = new LifecycleManager(); sysManager.add(sysInjector); sysManager.start(); + index = sysInjector.getInstance(IndexCollection.class).getSearchIndex(); + index.deleteAll(); int result = indexAll(); - writeVersion(); + index.markReady(); sysManager.stop(); dbManager.stop(); @@ -142,7 +151,7 @@ public class Reindex extends SiteProgram { AbstractModule changeIndexModule; switch (IndexModule.getIndexType(dbInjector)) { case LUCENE: - changeIndexModule = new LuceneIndexModule(false, threads, outputBase); + changeIndexModule = new LuceneIndexModule(version, threads, outputBase); break; case SOLR: changeIndexModule = new SolrIndexModule(false, threads, outputBase); @@ -207,11 +216,6 @@ public class Reindex extends SiteProgram { } } - private void deleteAll() throws IOException { - ChangeIndex index = sysInjector.getInstance(ChangeIndex.class); - index.deleteAll(); - } - private int indexAll() throws Exception { ReviewDb db = sysInjector.getInstance(ReviewDb.class); ListeningScheduledExecutorService executor = sysInjector.getInstance( @@ -464,10 +468,4 @@ public class Reindex extends SiteProgram { } } } - - private void writeVersion() throws IOException, - ConfigInvalidException { - ChangeIndex index = sysInjector.getInstance(ChangeIndex.class); - index.finishIndex(); - } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java index 7d7dcdb41c..1ffa6b5a40 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java @@ -21,8 +21,6 @@ import com.google.gerrit.server.query.QueryParseException; import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.ChangeDataSource; -import org.eclipse.jgit.errors.ConfigInvalidException; - import java.io.IOException; /** @@ -64,20 +62,27 @@ public interface ChangeIndex { } @Override - public ChangeDataSource getSource(Predicate p) - throws QueryParseException { + public ChangeDataSource getSource(Predicate p) { throw new UnsupportedOperationException(); } @Override - public void finishIndex() { + public void close() { // Do nothing. } + + @Override + public void markReady() { + throw new UnsupportedOperationException(); + } }; /** @return the schema version used by this index. */ public Schema getSchema(); + /** Close this index. */ + public void close(); + /** * Insert a change document into the index. *

@@ -137,11 +142,13 @@ public interface ChangeIndex { throws QueryParseException; /** - * Mark completion of indexing. + * Mark this index as up-to-date and ready to serve reads. + *

+ * Should only be called immediately after a reindex, either during an online + * schema upgrade while actively writing to this index, or during an offline + * reindex. * - * @throws ConfigInvalidException * @throws IOException */ - public void finishIndex() throws IOException, - ConfigInvalidException; + public void markReady() throws IOException; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java index 6094df167d..87a4df1602 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java @@ -15,9 +15,9 @@ package com.google.gerrit.server.index; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.gerrit.server.query.change.ChangeData; @@ -58,23 +58,16 @@ public class ChangeSchemas { return new Schema(false, Arrays.asList(fields)); } - private static final ImmutableMap> ALL; + public static final ImmutableMap> ALL; - public Schema get(int version) { + public static Schema get(int version) { Schema schema = ALL.get(version); checkArgument(schema != null, "Unrecognized schema version: %s", version); return schema; } - public static Schema getLatestRelease() { - Schema latest = null; - for (Schema schema : ALL.values()) { - if (schema.isRelease()) { - latest = schema; - } - } - checkState(latest != null, "No released schema versions found"); - return latest; + public static Schema getLatest() { + return Iterables.getLast(ALL.values()); } static { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java index f07bf4903f..a6cd2269c8 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java @@ -16,6 +16,7 @@ package com.google.gerrit.server.index; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import com.google.gerrit.extensions.events.LifecycleListener; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -28,7 +29,7 @@ import javax.annotation.Nullable; /** Dynamic pointers to the index versions used for searching and writing. */ @Singleton -public class IndexCollection { +public class IndexCollection implements LifecycleListener { private final CopyOnWriteArrayList writeIndexes; private final AtomicReference searchIndex; @@ -66,4 +67,21 @@ public class IndexCollection { } writeIndexes.add(index); } + + @Override + public void start() { + } + + @Override + public void stop() { + ChangeIndex read = searchIndex.get(); + if (read != null) { + read.close(); + } + for (ChangeIndex write : writeIndexes) { + if (write != read) { + write.close(); + } + } + } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java index a04fe80282..e96791a536 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java @@ -16,11 +16,11 @@ package com.google.gerrit.server.index; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.gerrit.lifecycle.LifecycleModule; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.git.WorkQueue.Executor; import com.google.gerrit.server.query.change.ChangeQueryRewriter; -import com.google.inject.AbstractModule; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Provides; @@ -34,7 +34,7 @@ import org.eclipse.jgit.lib.Config; * This module should not be used directly except by specific secondary indexer * implementations (e.g. Lucene). */ -public class IndexModule extends AbstractModule { +public class IndexModule extends LifecycleModule { public enum IndexType { SQL, LUCENE, SOLR; } @@ -57,6 +57,8 @@ public class IndexModule extends AbstractModule { bind(ChangeIndexer.class).to(ChangeIndexerImpl.class); bind(ChangeQueryRewriter.class).to(IndexRewriteImpl.class); bind(IndexRewriteImpl.BasicRewritesImpl.class); + bind(IndexCollection.class); + listener().to(IndexCollection.class); } @Provides diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java index 2bea308e18..9f4376b2dc 100644 --- a/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java +++ b/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java @@ -90,7 +90,11 @@ public class IndexRewriteTest extends TestCase { } @Override - public void finishIndex() { + public void close() { + } + + @Override + public void markReady() { throw new UnsupportedOperationException(); } } diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java index ea33b05ddf..7e32bac601 100644 --- a/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java @@ -31,8 +31,8 @@ import java.util.Map; class IndexVersionCheck implements LifecycleListener { public static final Map SCHEMA_VERSIONS = ImmutableMap.of( - SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(), - SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion()); + SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatest().getVersion(), + SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatest().getVersion()); public static File solrIndexConfig(SitePaths sitePaths) { return new File(sitePaths.index_dir, "gerrit_index.config"); diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java index b338dffbe3..b455c55757 100644 --- a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java @@ -52,7 +52,6 @@ import org.apache.solr.client.solrj.impl.CloudSolrServer; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; -import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.util.FS; @@ -120,6 +119,11 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener { return schema; } + @Override + public void close() { + stop(); + } + @Override public ListenableFuture insert(ChangeData cd) throws IOException { String id = cd.getId().toString(); @@ -321,8 +325,7 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener { } @Override - public void finishIndex() throws IOException, - ConfigInvalidException { + public void markReady() throws IOException { // TODO Move the schema version information to a special meta-document FileBasedConfig cfg = new FileBasedConfig( solrIndexConfig(sitePaths), diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java index 4b892a8b43..8c614f7d2c 100644 --- a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java @@ -61,6 +61,6 @@ public class SolrIndexModule extends LifecycleModule { IndexCollection indexes, FillArgs fillArgs) throws IOException { return new SolrChangeIndex(cfg, fillArgs, sitePaths, indexes, - ChangeSchemas.getLatestRelease(), base); + ChangeSchemas.getLatest(), base); } }