diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index f87b328add..2494e2cb68 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -245,11 +245,10 @@ public class LuceneChangeIndex implements ChangeIndex { } @Override - public void markReady() throws IOException { + public void markReady(boolean ready) throws IOException { try { FileBasedConfig cfg = LuceneVersionManager.loadGerritIndexConfig(sitePaths); - cfg.setBoolean("index", Integer.toString(schema.getVersion()), "ready", - true); + LuceneVersionManager.setReady(cfg, schema.getVersion(), ready); cfg.save(); } catch (ConfigInvalidException e) { throw new IOException(e); diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java index c63eb24c07..2f7cfa6e99 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java @@ -78,12 +78,12 @@ class LuceneVersionManager implements LifecycleListener { return cfg; } - private static boolean getReady(Config cfg, int version) { - return cfg.getBoolean("index", Integer.toString(version), "ready", false); + static void setReady(Config cfg, int version, boolean ready) { + cfg.setBoolean("index", Integer.toString(version), "ready", ready); } - private static void setReady(Config cfg, int version, boolean ready) { - cfg.setBoolean("index", Integer.toString(version), "ready", ready); + private static boolean getReady(Config cfg, int version) { + return cfg.getBoolean("index", Integer.toString(version), "ready", false); } private final SitePaths sitePaths; diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java index 6d47124834..f740fba97f 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java @@ -16,17 +16,8 @@ package com.google.gerrit.pgm; import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER; -import com.google.common.base.Stopwatch; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.lifecycle.LifecycleManager; @@ -38,20 +29,14 @@ import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.cache.CacheRemovalListener; import com.google.gerrit.server.cache.h2.DefaultCacheFactory; -import com.google.gerrit.server.git.GitRepositoryManager; -import com.google.gerrit.server.git.MultiProgressMonitor; -import com.google.gerrit.server.git.MultiProgressMonitor.Task; +import com.google.gerrit.server.index.ChangeBatchIndexer; import com.google.gerrit.server.index.ChangeIndex; -import com.google.gerrit.server.index.ChangeIndexer; import com.google.gerrit.server.index.ChangeSchemas; import com.google.gerrit.server.index.IndexCollection; -import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.NoIndexModule; import com.google.gerrit.server.patch.PatchListCacheImpl; -import com.google.gerrit.server.patch.PatchListLoader; -import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.solr.SolrIndexModule; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.SchemaFactory; @@ -63,39 +48,17 @@ import com.google.inject.Provider; import com.google.inject.ProvisionException; import com.google.inject.TypeLiteral; -import org.eclipse.jgit.diff.DiffEntry; -import org.eclipse.jgit.diff.DiffFormatter; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.RepositoryCache; import org.eclipse.jgit.lib.TextProgressMonitor; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.io.DisabledOutputStream; +import org.eclipse.jgit.util.io.NullOutputStream; import org.kohsuke.args4j.Option; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; public class Reindex extends SiteProgram { - private static final Logger log = LoggerFactory.getLogger(Reindex.class); - @Option(name = "--threads", usage = "Number of threads to use for indexing") private int threads = Runtime.getRuntime().availableProcessors(); @@ -136,9 +99,10 @@ public class Reindex extends SiteProgram { sysManager.start(); index = sysInjector.getInstance(IndexCollection.class).getSearchIndex(); + index.markReady(false); index.deleteAll(); int result = indexAll(); - index.markReady(); + index.markReady(true); sysManager.stop(); dbManager.stop(); @@ -218,9 +182,6 @@ public class Reindex extends SiteProgram { private int indexAll() throws Exception { ReviewDb db = sysInjector.getInstance(ReviewDb.class); - ListeningScheduledExecutorService executor = sysInjector.getInstance( - Key.get(ListeningScheduledExecutorService.class, IndexExecutor.class)); - ProgressMonitor pm = new TextProgressMonitor(); pm.start(1); pm.beginTask("Collecting projects", ProgressMonitor.UNKNOWN); @@ -238,234 +199,14 @@ public class Reindex extends SiteProgram { } pm.endTask(); - final MultiProgressMonitor mpm = - new MultiProgressMonitor(System.err, "Reindexing changes"); - final Task projTask = mpm.beginSubTask("projects", projects.size()); - final Task doneTask = mpm.beginSubTask(null, changeCount); - final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN); - - Stopwatch sw = new Stopwatch().start(); - final List> futures = - Lists.newArrayListWithCapacity(projects.size()); - final AtomicBoolean ok = new AtomicBoolean(true); - - for (final Project.NameKey project : projects) { - final ListenableFuture future = executor.submit( - new ReindexProject(project, doneTask, failedTask)); - futures.add(future); - future.addListener(new Runnable() { - @Override - public void run() { - try { - future.get(); - } catch (InterruptedException e) { - fail(project, e); - } catch (ExecutionException e) { - ok.set(false); // Logged by indexer. - } catch (RuntimeException e) { - failAndThrow(project, e); - } catch (Error e) { - failAndThrow(project, e); - } finally { - projTask.update(1); - } - } - - private void fail(Project.NameKey project, Throwable t) { - log.error("Failed to index project " + project, t); - ok.set(false); - } - - private void failAndThrow(Project.NameKey project, RuntimeException e) { - fail(project, e); - throw e; - } - - private void failAndThrow(Project.NameKey project, Error e) { - fail(project, e); - throw e; - } - }, MoreExecutors.sameThreadExecutor()); - } - - mpm.waitFor(Futures.transform(Futures.successfulAsList(futures), - new AsyncFunction, Void>() { - @Override - public ListenableFuture apply(List input) throws Exception { - mpm.end(); - return Futures.immediateFuture(null); - } - })); - double elapsed = sw.elapsed(TimeUnit.MILLISECONDS) / 1000d; - int n = doneTask.getCount() + failedTask.getCount(); - System.out.format("Reindexed %d changes in %.01fs (%.01f/s)\n", - n, elapsed, n/elapsed); - - return ok.get() ? 0 : 1; - } - - private class ReindexProject implements Callable { - private final ChangeIndexer indexer; - private final Project.NameKey project; - private final ListMultimap byId; - private final Task done; - private final Task failed; - private Repository repo; - private RevWalk walk; - - private ReindexProject(Project.NameKey project, Task done, Task failed) { - this.indexer = sysInjector.getInstance(ChangeIndexer.class); - this.project = project; - this.byId = ArrayListMultimap.create(); - this.done = done; - this.failed = failed; - } - - @Override - public Void call() throws Exception { - ReviewDb db = sysInjector.getInstance(ReviewDb.class); - GitRepositoryManager mgr = sysInjector.getInstance(GitRepositoryManager.class); - repo = mgr.openRepository(project); - - try { - Map refs = repo.getAllRefs(); - for (Change c : db.changes().byProject(project)) { - String refName = c.currentPatchSetId().toRefName(); - Ref r = refs.get(refName); - if (r != null) { - byId.put(r.getObjectId(), new ChangeData(c)); - } else { - fail("Failed to index change " + c.getId() - + " (" + refName + " not found)", true, null); - } - } - walk(); - } finally { - repo.close(); - RepositoryCache.close(repo); // Only used once per Reindex call. - } - return null; - } - - private void walk() throws Exception { - walk = new RevWalk(repo); - try { - // Walk only refs first to cover as many changes as we can without having - // to mark every single change. - for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) { - RevObject o = walk.parseAny(ref.getObjectId()); - if (o instanceof RevCommit) { - walk.markStart((RevCommit) o); - } - } - - RevCommit bCommit; - while ((bCommit = walk.next()) != null && !byId.isEmpty()) { - if (byId.containsKey(bCommit)) { - getPathsAndIndex(bCommit); - byId.removeAll(bCommit); - } - } - - for (ObjectId id : byId.keySet()) { - getPathsAndIndex(walk.parseCommit(id)); - } - } finally { - walk.release(); - } - } - - private void getPathsAndIndex(RevCommit bCommit) throws Exception { - RevTree bTree = bCommit.getTree(); - List cds = Lists.newArrayList(byId.get(bCommit)); - try { - RevTree aTree = aFor(bCommit, walk); - DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE); - try { - df.setRepository(repo); - if (!cds.isEmpty()) { - List paths = (aTree != null) - ? getPaths(df.scan(aTree, bTree)) - : Collections.emptyList(); - Iterator cdit = cds.iterator(); - for (ChangeData cd ; cdit.hasNext(); cdit.remove()) { - cd = cdit.next(); - try { - cd.setCurrentFilePaths(paths); - indexer.indexTask(cd).call(); - done.update(1); - if (verbose) { - System.out.println("Reindexed change " + cd.getId()); - } - } catch (Exception e) { - fail("Failed to index change " + cd.getId(), true, e); - } - } - } - } finally { - df.release(); - } - } catch (Exception e) { - fail("Failed to index commit " + bCommit.name(), false, e); - for (ChangeData cd : cds) { - fail("Failed to index change " + cd.getId(), true, null); - } - } - } - - private List getPaths(List filenames) { - Set paths = Sets.newTreeSet(); - for (DiffEntry e : filenames) { - if (e.getOldPath() != null) { - paths.add(e.getOldPath()); - } - if (e.getNewPath() != null) { - paths.add(e.getNewPath()); - } - } - return ImmutableList.copyOf(paths); - } - - private RevTree aFor(RevCommit b, RevWalk walk) throws IOException { - switch (b.getParentCount()) { - case 0: - return walk.parseTree(emptyTree()); - case 1: - RevCommit a = b.getParent(0); - walk.parseBody(a); - return walk.parseTree(a.getTree()); - case 2: - return PatchListLoader.automerge(repo, walk, b); - default: - return null; - } - } - - private ObjectId emptyTree() throws IOException { - ObjectInserter oi = repo.newObjectInserter(); - try { - ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {}); - oi.flush(); - return id; - } finally { - oi.release(); - } - } - - private void fail(String error, boolean failed, Exception e) { - if (failed) { - this.failed.update(1); - } - - if (e != null) { - log.warn(error, e); - } else { - log.warn(error); - } - - if (verbose) { - System.out.println(error); - } - } + ChangeBatchIndexer batchIndexer = + sysInjector.getInstance(ChangeBatchIndexer.class); + ChangeBatchIndexer.Result result = batchIndexer.indexAll( + index, projects, projects.size(), changeCount, System.err, + verbose ? System.out : NullOutputStream.INSTANCE); + int n = result.doneCount() + result.failedCount(); + double t = result.elapsed(TimeUnit.MILLISECONDS) / 1000d; + System.out.format("Reindexed %d changes in %.01fs (%.01f/s)\n", n, t, n/t); + return result.success() ? 0 : 1; } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeBatchIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeBatchIndexer.java new file mode 100644 index 0000000000..faecee9478 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeBatchIndexer.java @@ -0,0 +1,359 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package com.google.gerrit.server.git; + +package com.google.gerrit.server.index; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.AsyncFunction; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gerrit.reviewdb.client.Change; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.server.ReviewDb; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gerrit.server.git.MultiProgressMonitor; +import com.google.gerrit.server.git.MultiProgressMonitor.Task; +import com.google.gerrit.server.patch.PatchListLoader; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.inject.Inject; +import com.google.inject.Provider; + +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.util.io.DisabledOutputStream; +import org.eclipse.jgit.util.io.NullOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ChangeBatchIndexer { + private static final Logger log = + LoggerFactory.getLogger(ChangeBatchIndexer.class); + + public static class Result { + private final long elapsedNanos; + private final boolean success; + private final int done; + private final int failed; + + private Result(Stopwatch sw, boolean success, int done, int failed) { + this.elapsedNanos = sw.elapsed(TimeUnit.NANOSECONDS); + this.success = success; + this.done = done; + this.failed = failed; + } + + public boolean success() { + return success; + } + + public int doneCount() { + return done; + } + + public int failedCount() { + return failed; + } + + public long elapsed(TimeUnit timeUnit) { + return timeUnit.convert(elapsedNanos, TimeUnit.NANOSECONDS); + } + } + + private final Provider dbProvider; + private final GitRepositoryManager repoManager; + private final ListeningScheduledExecutorService executor; + private final ChangeIndexer.Factory indexerFactory; + + @Inject + ChangeBatchIndexer(Provider dbProvider, + GitRepositoryManager repoManager, + @IndexExecutor ListeningScheduledExecutorService executor, + ChangeIndexer.Factory indexerFactory) { + this.dbProvider = dbProvider; + this.repoManager = repoManager; + this.executor = executor; + this.indexerFactory = indexerFactory; + } + + public Result indexAll(ChangeIndex index, Iterable projects, + int numProjects, int numChanges, OutputStream progressOut, + OutputStream verboseOut) { + if (progressOut == null) { + progressOut = NullOutputStream.INSTANCE; + } + PrintWriter verboseWriter = verboseOut != null ? new PrintWriter(verboseOut) + : null; + + Stopwatch sw = new Stopwatch().start(); + final MultiProgressMonitor mpm = + new MultiProgressMonitor(progressOut, "Reindexing changes"); + final Task projTask = mpm.beginSubTask("projects", + numProjects >= 0 ? numProjects : MultiProgressMonitor.UNKNOWN); + final Task doneTask = mpm.beginSubTask(null, + numChanges >= 0 ? numChanges : MultiProgressMonitor.UNKNOWN); + final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN); + + final List> futures = Lists.newArrayList(); + final AtomicBoolean ok = new AtomicBoolean(true); + + for (final Project.NameKey project : projects) { + final ListenableFuture future = executor.submit(new ReindexProject( + indexerFactory.create(index), project, doneTask, failedTask, + verboseWriter)); + futures.add(future); + future.addListener(new Runnable() { + @Override + public void run() { + try { + future.get(); + } catch (InterruptedException e) { + fail(project, e); + } catch (ExecutionException e) { + ok.set(false); // Logged by indexer. + } catch (RuntimeException e) { + failAndThrow(project, e); + } catch (Error e) { + failAndThrow(project, e); + } finally { + projTask.update(1); + } + } + + private void fail(Project.NameKey project, Throwable t) { + log.error("Failed to index project " + project, t); + ok.set(false); + } + + private void failAndThrow(Project.NameKey project, RuntimeException e) { + fail(project, e); + throw e; + } + + private void failAndThrow(Project.NameKey project, Error e) { + fail(project, e); + throw e; + } + }, MoreExecutors.sameThreadExecutor()); + } + + try { + mpm.waitFor(Futures.transform(Futures.successfulAsList(futures), + new AsyncFunction, Void>() { + @Override + public ListenableFuture apply(List input) { + mpm.end(); + return Futures.immediateFuture(null); + } + })); + } catch (ExecutionException e) { + log.error("Error in batch indexer", e); + ok.set(false); + } + return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount()); + } + + private class ReindexProject implements Callable { + private final ChangeIndexer indexer; + private final Project.NameKey project; + private final ListMultimap byId; + private final Task done; + private final Task failed; + private final PrintWriter verboseWriter; + private Repository repo; + private RevWalk walk; + + private ReindexProject(ChangeIndexer indexer, Project.NameKey project, + Task done, Task failed, PrintWriter verboseWriter) { + this.indexer = indexer; + this.project = project; + this.byId = ArrayListMultimap.create(); + this.done = done; + this.verboseWriter = verboseWriter; + this.failed = failed; + } + + @Override + public Void call() throws Exception { + ReviewDb db = dbProvider.get(); + repo = repoManager.openRepository(project); + + try { + Map refs = repo.getAllRefs(); + for (Change c : db.changes().byProject(project)) { + Ref r = refs.get(c.currentPatchSetId().toRefName()); + if (r != null) { + byId.put(r.getObjectId(), new ChangeData(c)); + } + } + walk(); + } finally { + repo.close(); + // TODO(dborowitz): Opening all repositories in a live server may be + // wasteful; see if we can determine which ones it is safe to close with + // RepositoryCache.close(repo). + } + return null; + } + + private void walk() throws Exception { + walk = new RevWalk(repo); + try { + // Walk only refs first to cover as many changes as we can without having + // to mark every single change. + for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) { + RevObject o = walk.parseAny(ref.getObjectId()); + if (o instanceof RevCommit) { + walk.markStart((RevCommit) o); + } + } + + RevCommit bCommit; + while ((bCommit = walk.next()) != null && !byId.isEmpty()) { + if (byId.containsKey(bCommit)) { + getPathsAndIndex(bCommit); + byId.removeAll(bCommit); + } + } + + for (ObjectId id : byId.keySet()) { + getPathsAndIndex(walk.parseCommit(id)); + } + } finally { + walk.release(); + } + } + + private void getPathsAndIndex(RevCommit bCommit) throws Exception { + RevTree bTree = bCommit.getTree(); + List cds = Lists.newArrayList(byId.get(bCommit)); + try { + RevTree aTree = aFor(bCommit, walk); + DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE); + try { + df.setRepository(repo); + if (!cds.isEmpty()) { + List paths = (aTree != null) + ? getPaths(df.scan(aTree, bTree)) + : Collections.emptyList(); + Iterator cdit = cds.iterator(); + for (ChangeData cd ; cdit.hasNext(); cdit.remove()) { + cd = cdit.next(); + try { + cd.setCurrentFilePaths(paths); + indexer.indexTask(cd).call(); + done.update(1); + if (verboseWriter != null) { + verboseWriter.println("Reindexed change " + cd.getId()); + } + } catch (Exception e) { + fail("Failed to index change " + cd.getId(), true, e); + } + } + } + } finally { + df.release(); + } + } catch (Exception e) { + fail("Failed to index commit " + bCommit.name(), false, e); + for (ChangeData cd : cds) { + fail("Failed to index change " + cd.getId(), true, null); + } + } + } + + private List getPaths(List filenames) { + Set paths = Sets.newTreeSet(); + for (DiffEntry e : filenames) { + if (e.getOldPath() != null) { + paths.add(e.getOldPath()); + } + if (e.getNewPath() != null) { + paths.add(e.getNewPath()); + } + } + return ImmutableList.copyOf(paths); + } + + private RevTree aFor(RevCommit b, RevWalk walk) throws IOException { + switch (b.getParentCount()) { + case 0: + return walk.parseTree(emptyTree()); + case 1: + RevCommit a = b.getParent(0); + walk.parseBody(a); + return walk.parseTree(a.getTree()); + case 2: + return PatchListLoader.automerge(repo, walk, b); + default: + return null; + } + } + + private ObjectId emptyTree() throws IOException { + ObjectInserter oi = repo.newObjectInserter(); + try { + ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {}); + oi.flush(); + return id; + } finally { + oi.release(); + } + } + + private void fail(String error, boolean failed, Exception e) { + if (failed) { + this.failed.update(1); + } + + if (e != null) { + log.warn(error, e); + } else { + log.warn(error); + } + + if (verboseWriter != null) { + verboseWriter.println(error); + } + } + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java index 1ffa6b5a40..e41195613a 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java @@ -72,7 +72,7 @@ public interface ChangeIndex { } @Override - public void markReady() { + public void markReady(boolean ready) { throw new UnsupportedOperationException(); } }; @@ -142,13 +142,10 @@ public interface ChangeIndex { throws QueryParseException; /** - * Mark this index as up-to-date and ready to serve reads. - *

- * Should only be called immediately after a reindex, either during an online - * schema upgrade while actively writing to this index, or during an offline - * reindex. + * Mark whether this index is up-to-date and ready to serve reads. * + * @param ready whether the index is ready * @throws IOException */ - public void markReady() throws IOException; + public void markReady(boolean ready) throws IOException; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexer.java index c25174e661..0aa3fc8fee 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexer.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexer.java @@ -29,6 +29,11 @@ import java.util.concurrent.Callable; * compute some of the fields and/or update the index. */ public abstract class ChangeIndexer { + public interface Factory { + ChangeIndexer create(ChangeIndex index); + ChangeIndexer create(IndexCollection indexes); + } + /** Instance indicating secondary index is disabled. */ public static final ChangeIndexer DISABLED = new ChangeIndexer(null) { @Override diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexerImpl.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexerImpl.java index c6325a3c53..1de5c9910d 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexerImpl.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndexerImpl.java @@ -21,9 +21,10 @@ import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.util.RequestContext; import com.google.gerrit.server.util.ThreadLocalRequestContext; import com.google.gwtorm.server.SchemaFactory; -import com.google.inject.Inject; import com.google.inject.OutOfScopeException; import com.google.inject.Provider; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; import com.google.inject.util.Providers; import org.slf4j.Logger; @@ -42,18 +43,32 @@ public class ChangeIndexerImpl extends ChangeIndexer { LoggerFactory.getLogger(ChangeIndexerImpl.class); private final IndexCollection indexes; + private final ChangeIndex index; private final SchemaFactory schemaFactory; private final ThreadLocalRequestContext context; - @Inject + @AssistedInject ChangeIndexerImpl(@IndexExecutor ListeningScheduledExecutorService executor, - IndexCollection indexes, SchemaFactory schemaFactory, - ThreadLocalRequestContext context) { + ThreadLocalRequestContext context, + @Assisted ChangeIndex index) { super(executor); - this.indexes = indexes; this.schemaFactory = schemaFactory; this.context = context; + this.index = index; + this.indexes = null; + } + + @AssistedInject + ChangeIndexerImpl(@IndexExecutor ListeningScheduledExecutorService executor, + SchemaFactory schemaFactory, + ThreadLocalRequestContext context, + @Assisted IndexCollection indexes) { + super(executor); + this.schemaFactory = schemaFactory; + this.context = context; + this.index = null; + this.indexes = indexes; } @Override @@ -84,8 +99,12 @@ public class ChangeIndexerImpl extends ChangeIndexer { throw new OutOfScopeException("No user during ChangeIndexer"); } }); - for (ChangeIndex index : indexes.getWriteIndexes()) { - index.replace(cd); // TODO(dborowitz): Parallelize these + if (indexes != null) { + for (ChangeIndex i : indexes.getWriteIndexes()) { + i.replace(cd); // TODO(dborowitz): Parallelize these + } + } else { + index.replace(cd); } return null; } finally { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java index a6cd2269c8..6a2c99cb6a 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java @@ -68,6 +68,14 @@ public class IndexCollection implements LifecycleListener { writeIndexes.add(index); } + public void removeWriteIndex(int version) { + writeIndexes.remove(version); + } + + public ChangeIndex getWriteIndex(int version) { + return writeIndexes.get(version); + } + @Override public void start() { } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java index e96791a536..f46fe7c715 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java @@ -25,6 +25,7 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Provides; import com.google.inject.Singleton; +import com.google.inject.assistedinject.FactoryModuleBuilder; import org.eclipse.jgit.lib.Config; @@ -54,11 +55,13 @@ public class IndexModule extends LifecycleModule { @Override protected void configure() { - bind(ChangeIndexer.class).to(ChangeIndexerImpl.class); bind(ChangeQueryRewriter.class).to(IndexRewriteImpl.class); bind(IndexRewriteImpl.BasicRewritesImpl.class); bind(IndexCollection.class); listener().to(IndexCollection.class); + install(new FactoryModuleBuilder() + .implement(ChangeIndexer.class, ChangeIndexerImpl.class) + .build(ChangeIndexer.Factory.class)); } @Provides @@ -79,4 +82,11 @@ public class IndexModule extends LifecycleModule { } return MoreExecutors.listeningDecorator(executor); } + + @Provides + ChangeIndexer getChangeIndexer( + ChangeIndexer.Factory factory, + IndexCollection indexes) { + return factory.create(indexes); + } } diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java index 9f4376b2dc..e91c3a069a 100644 --- a/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java +++ b/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java @@ -94,7 +94,7 @@ public class IndexRewriteTest extends TestCase { } @Override - public void markReady() { + public void markReady(boolean ready) { throw new UnsupportedOperationException(); } } diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java index b455c55757..b95978eaac 100644 --- a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java @@ -325,13 +325,14 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener { } @Override - public void markReady() throws IOException { + public void markReady(boolean ready) throws IOException { // TODO Move the schema version information to a special meta-document FileBasedConfig cfg = new FileBasedConfig( solrIndexConfig(sitePaths), FS.detect()); for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { - cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue()); + cfg.setInt("index", e.getKey(), "schemaVersion", + ready ? e.getValue() : -1); } cfg.save(); }