Reindex: add --dry-run flag
We are working on evaluating performance of Lucene indexing, and want to be able to separate the cost of Gerrit-specific operations from the cost of Lucene operations. This flag prevents any writing to the Lucene indexes, which will help with that. We may want to remove this flag eventually. Change-Id: I0e68f26c5d89c1c9b1b0b5d082af3b9241ce8d80
This commit is contained in:
@@ -43,8 +43,6 @@ import com.google.gerrit.server.query.change.ChangeDataSource;
|
|||||||
import com.google.gerrit.server.query.change.IndexRewriteImpl;
|
import com.google.gerrit.server.query.change.IndexRewriteImpl;
|
||||||
import com.google.gwtorm.server.OrmException;
|
import com.google.gwtorm.server.OrmException;
|
||||||
import com.google.gwtorm.server.ResultSet;
|
import com.google.gwtorm.server.ResultSet;
|
||||||
import com.google.inject.Inject;
|
|
||||||
import com.google.inject.Singleton;
|
|
||||||
|
|
||||||
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.document.Field;
|
import org.apache.lucene.document.Field;
|
||||||
@@ -76,19 +74,20 @@ import java.util.Set;
|
|||||||
* though there may be some lag between a committed write and it showing up to
|
* though there may be some lag between a committed write and it showing up to
|
||||||
* other threads' searchers.
|
* other threads' searchers.
|
||||||
*/
|
*/
|
||||||
@Singleton
|
|
||||||
public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
||||||
public static final Version LUCENE_VERSION = Version.LUCENE_43;
|
public static final Version LUCENE_VERSION = Version.LUCENE_43;
|
||||||
public static final String CHANGES_OPEN = "changes_open";
|
public static final String CHANGES_OPEN = "changes_open";
|
||||||
public static final String CHANGES_CLOSED = "changes_closed";
|
public static final String CHANGES_CLOSED = "changes_closed";
|
||||||
|
|
||||||
private final FillArgs fillArgs;
|
private final FillArgs fillArgs;
|
||||||
|
private final boolean readOnly;
|
||||||
private final SubIndex openIndex;
|
private final SubIndex openIndex;
|
||||||
private final SubIndex closedIndex;
|
private final SubIndex closedIndex;
|
||||||
|
|
||||||
@Inject
|
LuceneChangeIndex(SitePaths sitePaths, FillArgs fillArgs, boolean readOnly)
|
||||||
LuceneChangeIndex(SitePaths sitePaths, FillArgs fillArgs) throws IOException {
|
throws IOException {
|
||||||
this.fillArgs = fillArgs;
|
this.fillArgs = fillArgs;
|
||||||
|
this.readOnly = readOnly;
|
||||||
openIndex = new SubIndex(new File(sitePaths.index_dir, CHANGES_OPEN));
|
openIndex = new SubIndex(new File(sitePaths.index_dir, CHANGES_OPEN));
|
||||||
closedIndex = new SubIndex(new File(sitePaths.index_dir, CHANGES_CLOSED));
|
closedIndex = new SubIndex(new File(sitePaths.index_dir, CHANGES_CLOSED));
|
||||||
}
|
}
|
||||||
@@ -108,6 +107,9 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
public void insert(ChangeData cd) throws IOException {
|
public void insert(ChangeData cd) throws IOException {
|
||||||
Term id = idTerm(cd);
|
Term id = idTerm(cd);
|
||||||
Document doc = toDocument(cd);
|
Document doc = toDocument(cd);
|
||||||
|
if (readOnly) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (cd.getChange().getStatus().isOpen()) {
|
if (cd.getChange().getStatus().isOpen()) {
|
||||||
closedIndex.delete(id);
|
closedIndex.delete(id);
|
||||||
openIndex.insert(doc);
|
openIndex.insert(doc);
|
||||||
@@ -121,6 +123,9 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
public void replace(ChangeData cd) throws IOException {
|
public void replace(ChangeData cd) throws IOException {
|
||||||
Term id = idTerm(cd);
|
Term id = idTerm(cd);
|
||||||
Document doc = toDocument(cd);
|
Document doc = toDocument(cd);
|
||||||
|
if (readOnly) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (cd.getChange().getStatus().isOpen()) {
|
if (cd.getChange().getStatus().isOpen()) {
|
||||||
closedIndex.delete(id);
|
closedIndex.delete(id);
|
||||||
openIndex.replace(id, doc);
|
openIndex.replace(id, doc);
|
||||||
@@ -133,6 +138,9 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
@Override
|
@Override
|
||||||
public void delete(ChangeData cd) throws IOException {
|
public void delete(ChangeData cd) throws IOException {
|
||||||
Term id = idTerm(cd);
|
Term id = idTerm(cd);
|
||||||
|
if (readOnly) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (cd.getChange().getStatus().isOpen()) {
|
if (cd.getChange().getStatus().isOpen()) {
|
||||||
openIndex.delete(id);
|
openIndex.delete(id);
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -15,20 +15,29 @@
|
|||||||
package com.google.gerrit.lucene;
|
package com.google.gerrit.lucene;
|
||||||
|
|
||||||
import com.google.gerrit.lifecycle.LifecycleModule;
|
import com.google.gerrit.lifecycle.LifecycleModule;
|
||||||
|
import com.google.gerrit.server.config.SitePaths;
|
||||||
import com.google.gerrit.server.index.ChangeIndex;
|
import com.google.gerrit.server.index.ChangeIndex;
|
||||||
|
import com.google.gerrit.server.index.FieldDef.FillArgs;
|
||||||
import com.google.gerrit.server.index.IndexModule;
|
import com.google.gerrit.server.index.IndexModule;
|
||||||
|
import com.google.inject.Provides;
|
||||||
|
import com.google.inject.Singleton;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
public class LuceneIndexModule extends LifecycleModule {
|
public class LuceneIndexModule extends LifecycleModule {
|
||||||
private final boolean checkVersion;
|
private final boolean checkVersion;
|
||||||
private final int threads;
|
private final int threads;
|
||||||
|
private final boolean readOnly;
|
||||||
|
|
||||||
public LuceneIndexModule() {
|
public LuceneIndexModule() {
|
||||||
this(true, 0);
|
this(true, 0, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public LuceneIndexModule(boolean checkVersion, int threads) {
|
public LuceneIndexModule(boolean checkVersion, int threads,
|
||||||
|
boolean readOnly) {
|
||||||
this.checkVersion = checkVersion;
|
this.checkVersion = checkVersion;
|
||||||
this.threads = threads;
|
this.threads = threads;
|
||||||
|
this.readOnly = readOnly;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -40,4 +49,11 @@ public class LuceneIndexModule extends LifecycleModule {
|
|||||||
listener().to(IndexVersionCheck.class);
|
listener().to(IndexVersionCheck.class);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Provides
|
||||||
|
@Singleton
|
||||||
|
public LuceneChangeIndex getChangeIndex(SitePaths sitePaths,
|
||||||
|
FillArgs fillArgs) throws IOException {
|
||||||
|
return new LuceneChangeIndex(sitePaths, fillArgs, readOnly);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,6 +72,9 @@ public class Reindex extends SiteProgram {
|
|||||||
@Option(name = "--threads", usage = "Number of threads to use for indexing")
|
@Option(name = "--threads", usage = "Number of threads to use for indexing")
|
||||||
private int threads = Runtime.getRuntime().availableProcessors();
|
private int threads = Runtime.getRuntime().availableProcessors();
|
||||||
|
|
||||||
|
@Option(name = "--dry-run", usage = "Dry run: don't write anything to index")
|
||||||
|
private boolean dryRun;
|
||||||
|
|
||||||
private Injector dbInjector;
|
private Injector dbInjector;
|
||||||
private Injector sysInjector;
|
private Injector sysInjector;
|
||||||
private SitePaths sitePaths;
|
private SitePaths sitePaths;
|
||||||
@@ -105,7 +108,7 @@ public class Reindex extends SiteProgram {
|
|||||||
private Injector createSysInjector() {
|
private Injector createSysInjector() {
|
||||||
List<Module> modules = Lists.newArrayList();
|
List<Module> modules = Lists.newArrayList();
|
||||||
modules.add(PatchListCacheImpl.module());
|
modules.add(PatchListCacheImpl.module());
|
||||||
modules.add(new LuceneIndexModule(false, threads));
|
modules.add(new LuceneIndexModule(false, threads, dryRun));
|
||||||
modules.add(new ReviewDbModule());
|
modules.add(new ReviewDbModule());
|
||||||
modules.add(new AbstractModule() {
|
modules.add(new AbstractModule() {
|
||||||
@SuppressWarnings("rawtypes")
|
@SuppressWarnings("rawtypes")
|
||||||
@@ -163,6 +166,9 @@ public class Reindex extends SiteProgram {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void deleteAll() throws IOException {
|
private void deleteAll() throws IOException {
|
||||||
|
if (dryRun) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
for (String index : SCHEMA_VERSIONS.keySet()) {
|
for (String index : SCHEMA_VERSIONS.keySet()) {
|
||||||
File file = new File(sitePaths.index_dir, index);
|
File file = new File(sitePaths.index_dir, index);
|
||||||
if (file.exists()) {
|
if (file.exists()) {
|
||||||
@@ -216,6 +222,9 @@ public class Reindex extends SiteProgram {
|
|||||||
|
|
||||||
private void writeVersion() throws IOException,
|
private void writeVersion() throws IOException,
|
||||||
ConfigInvalidException {
|
ConfigInvalidException {
|
||||||
|
if (dryRun) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
FileBasedConfig cfg =
|
FileBasedConfig cfg =
|
||||||
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
|
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
|
||||||
cfg.load();
|
cfg.load();
|
||||||
|
|||||||
Reference in New Issue
Block a user