opendev/gerrit
Code Issues Proposed changes

Scan index directory to determine Lucene versions

Browse Source 
Mark index versions as "ready" when they are fully indexed and the
server is not running (i.e. from Reindex). By default, search from the
most recent ready index version, and write to both the most recent
ready version and (if different) the most recent known version.

At server startup, mark all versions except those we are about to
start writing to as not ready.

Change-Id: Icf42a3eb27b0445899300d60941cd701a8072d41
This commit is contained in:
Dave Borowitz 2013-06-26 08:17:28 -06:00
parent f7cbbe8017
commit d103b2b61c
14 changed files with 372 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gerrit-lucene/BUCK
View File

@ -27,7 +27,9 @@ java_library(
'//gerrit-server:server',
'//lib:guava',
'//lib:gwtorm',
'//lib:jsr305',
'//lib/guice:guice',
'//lib/guice:guice-assistedinject',
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:analyzers-common',

92
gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java
View File

@ -1,92 +0,0 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
import static org.apache.lucene.util.Version.LUCENE_CURRENT;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
import org.apache.lucene.util.Version;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import java.io.File;
import java.io.IOException;
import java.util.Map;
public class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
LuceneChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
LuceneChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
public static File gerritIndexConfig(SitePaths sitePaths) {
return new File(sitePaths.index_dir, "gerrit_index.config");
}
private final SitePaths sitePaths;
@Inject
IndexVersionCheck(SitePaths sitePaths) {
this.sitePaths = sitePaths;
}
@Override
public void start() {
File file = gerritIndexConfig(sitePaths);
try {
FileBasedConfig cfg = new FileBasedConfig(file, FS.detect());
cfg.load();
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
if (schemaVersion != e.getValue()) {
throw new ProvisionException(String.format(
"wrong index schema version for \"%s\": expected %d, found %d%s",
e.getKey(), e.getValue(), schemaVersion, upgrade()));
}
}
@SuppressWarnings("deprecation")
Version luceneVersion =
cfg.getEnum("lucene", null, "version", LUCENE_CURRENT);
if (luceneVersion != LUCENE_VERSION) {
throw new ProvisionException(String.format(
"wrong Lucene version: expected %d, found %d%s",
luceneVersion, LUCENE_VERSION, upgrade()));
}
} catch (IOException e) {
throw new ProvisionException("unable to read " + file);
} catch (ConfigInvalidException e) {
throw new ProvisionException("invalid config file " + file);
}
}
@Override
public void stop() {
// Do nothing.
}
private final String upgrade() {
return "\nRun reindex to rebuild the index:\n"
+ "$ java -jar gerrit.war reindex -d "
+ sitePaths.site_path.getAbsolutePath();
}
}

64
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@ -14,8 +14,6 @@
package com.google.gerrit.lucene;
import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
@ -26,7 +24,6 @@ import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
@ -35,7 +32,6 @@ import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexRewriteImpl;
import com.google.gerrit.server.index.Schema;
@ -45,6 +41,8 @@ import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
@ -69,7 +67,6 @@ import org.apache.lucene.util.Version;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -79,11 +76,12 @@ import java.sql.Timestamp;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import javax.annotation.Nullable;
/**
* Secondary index implementation using Apache Lucene.
* <p>
@ -92,7 +90,7 @@ import java.util.concurrent.Future;
* though there may be some lag between a committed write and it showing up to
* other threads' searchers.
*/
public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
public class LuceneChangeIndex implements ChangeIndex {
private static final Logger log =
LoggerFactory.getLogger(LuceneChangeIndex.class);
@ -101,6 +99,10 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
public static final String CHANGES_CLOSED = "closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema, String base);
}
private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) {
IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION,
new StandardAnalyzer(LUCENE_VERSION));
@ -115,28 +117,27 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
private final SitePaths sitePaths;
private final FillArgs fillArgs;
private final IndexCollection indexes;
private final ExecutorService executor;
private final File dir;
private final Schema<ChangeData> schema;
private final SubIndex openIndex;
private final SubIndex closedIndex;
LuceneChangeIndex(@GerritServerConfig Config cfg,
@AssistedInject
LuceneChangeIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
IndexCollection indexes,
@IndexExecutor ListeningScheduledExecutorService executor,
FillArgs fillArgs,
Schema<ChangeData> schema,
String base) throws IOException {
this.indexes = indexes;
@Assisted Schema<ChangeData> schema,
@Assisted @Nullable String base) throws IOException {
this.sitePaths = sitePaths;
this.fillArgs = fillArgs;
this.executor = executor;
this.schema = schema;
File dir;
if (base == null) {
dir = new File(sitePaths.index_dir, "changes_" + schema.getVersion());
dir = LuceneVersionManager.getDir(sitePaths, schema);
} else {
dir = new File(base);
}
@ -147,13 +148,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
}
@Override
public void start() {
indexes.setSearchIndex(this);
indexes.addWriteIndex(this);
}
@Override
public void stop() {
public void close() {
List<Future<?>> closeFutures = Lists.newArrayListWithCapacity(2);
closeFutures.add(executor.submit(new Runnable() {
@Override
@ -249,6 +244,18 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
return new QuerySource(indexes, QueryBuilder.toQuery(p));
}
@Override
public void markReady() throws IOException {
try {
FileBasedConfig cfg = LuceneVersionManager.loadGerritIndexConfig(sitePaths);
cfg.setBoolean("index", Integer.toString(schema.getVersion()), "ready",
true);
cfg.save();
} catch (ConfigInvalidException e) {
throw new IOException(e);
}
}
private static class QuerySource implements ChangeDataSource {
// TODO(dborowitz): Push limit down from predicate tree.
private static final int LIMIT = 1000;
@ -388,17 +395,4 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
@Override
public void finishIndex() throws IOException,
ConfigInvalidException {
FileBasedConfig cfg =
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
}
cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
cfg.save();
}
}

82
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
View File

@ -14,57 +14,89 @@
package com.google.gerrit.lucene;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.FactoryModule;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import org.eclipse.jgit.lib.Config;
import java.io.IOException;
public class LuceneIndexModule extends LifecycleModule {
private final boolean checkVersion;
private final Integer singleVersion;
private final int threads;
private final String base;
public LuceneIndexModule() {
this(true, 0, null);
this(null, 0, null);
}
public LuceneIndexModule(boolean checkVersion, int threads,
public LuceneIndexModule(Integer singleVersion, int threads,
String base) {
this.checkVersion = checkVersion;
this.singleVersion = singleVersion;
this.threads = threads;
this.base = base;
}
@Override
protected void configure() {
install(new IndexModule(threads));
bind(ChangeIndex.class).to(LuceneChangeIndex.class);
listener().to(LuceneChangeIndex.class);
if (checkVersion) {
listener().to(IndexVersionCheck.class);
install(new FactoryModule() {
@Override
public void configure() {
factory(LuceneChangeIndex.Factory.class);
}
});
install(new IndexModule(threads));
if (singleVersion == null && base == null) {
listener().to(LuceneVersionManager.class);
} else {
install(new SingleVersionModule());
}
}
private class SingleVersionModule extends LifecycleModule {
@Override
public void configure() {
listener().to(SingleVersionListener.class);
}
@Provides
@Singleton
public LuceneChangeIndex getChangeIndex(@GerritServerConfig Config cfg,
SitePaths sitePaths,
IndexCollection indexes,
@IndexExecutor ListeningScheduledExecutorService executor,
FillArgs fillArgs) throws IOException {
return new LuceneChangeIndex(cfg, sitePaths, indexes, executor, fillArgs,
ChangeSchemas.getLatestRelease(), base);
LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory,
SitePaths sitePaths) {
Schema<ChangeData> schema = singleVersion != null
? ChangeSchemas.get(singleVersion)
: ChangeSchemas.getLatest();
return factory.create(schema, base);
}
}
@Singleton
static class SingleVersionListener implements LifecycleListener {
private final IndexCollection indexes;
private final LuceneChangeIndex index;
@Inject
SingleVersionListener(IndexCollection indexes,
LuceneChangeIndex index) {
this.indexes = indexes;
this.index = index;
}
@Override
public void start() {
indexes.setSearchIndex(index);
indexes.addWriteIndex(index);
}
@Override
public void stop() {
index.close();
}
}
}

209
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java Normal file
View File

@ -0,0 +1,209 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.lucene;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.TreeMap;
@Singleton
class LuceneVersionManager implements LifecycleListener {
private static final Logger log = LoggerFactory
.getLogger(LuceneVersionManager.class);
private static final String CHANGES_PREFIX = "changes_";
private static class Version {
private final Schema<ChangeData> schema;
private final int version;
private final boolean exists;
private final boolean ready;
private Version(Schema<ChangeData> schema, int version, boolean exists,
boolean ready) {
checkArgument(schema == null || schema.getVersion() == version);
this.schema = schema;
this.version = version;
this.exists = exists;
this.ready = ready;
}
}
static File getDir(SitePaths sitePaths, Schema<ChangeData> schema) {
return new File(sitePaths.index_dir, String.format("%s%04d",
CHANGES_PREFIX, schema.getVersion()));
}
static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths)
throws ConfigInvalidException, IOException {
FileBasedConfig cfg = new FileBasedConfig(
new File(sitePaths.index_dir, "gerrit_index.config"), FS.detect());
cfg.load();
return cfg;
}
private static boolean getReady(Config cfg, int version) {
return cfg.getBoolean("index", Integer.toString(version), "ready", false);
}
private static void setReady(Config cfg, int version, boolean ready) {
cfg.setBoolean("index", Integer.toString(version), "ready", ready);
}
private final SitePaths sitePaths;
private final LuceneChangeIndex.Factory indexFactory;
private final IndexCollection indexes;
@Inject
LuceneVersionManager(
SitePaths sitePaths,
LuceneChangeIndex.Factory indexFactory,
IndexCollection indexes) {
this.sitePaths = sitePaths;
this.indexFactory = indexFactory;
this.indexes = indexes;
}
@Override
public void start() {
FileBasedConfig cfg;
try {
cfg = loadGerritIndexConfig(sitePaths);
} catch (ConfigInvalidException e) {
throw fail(e);
} catch (IOException e) {
throw fail(e);
}
TreeMap<Integer, Version> versions = scanVersions(cfg);
// Search from the most recent ready version.
// Write to the most recent ready version and the most recent version.
Version search = null;
List<Version> write = Lists.newArrayListWithCapacity(2);
for (Version v : versions.descendingMap().values()) {
if (v.schema == null) {
continue;
}
if (write.isEmpty()) {
write.add(v);
}
if (v.ready) {
search = v;
if (!write.contains(v)) {
write.add(v);
}
break;
}
}
if (search == null) {
throw new ProvisionException("No index versions ready; run Reindex");
}
markNotReady(cfg, versions.values(), write);
LuceneChangeIndex searchIndex = indexFactory.create(search.schema, null);
indexes.setSearchIndex(searchIndex);
for (Version v : write) {
if (v.schema != null) {
if (v.version != search.version) {
indexes.addWriteIndex(indexFactory.create(v.schema, null));
} else {
indexes.addWriteIndex(searchIndex);
}
}
}
}
private TreeMap<Integer, Version> scanVersions(Config cfg) {
TreeMap<Integer, Version> versions = Maps.newTreeMap();
for (Schema<ChangeData> schema : ChangeSchemas.ALL.values()) {
File f = getDir(sitePaths, schema);
boolean exists = f.exists() && f.isDirectory();
if (exists && !f.isDirectory()) {
log.warn("Not a directory: %s", f.getAbsolutePath());
}
int v = schema.getVersion();
versions.put(v, new Version(schema, v, exists, getReady(cfg, v)));
}
for (File f : sitePaths.index_dir.listFiles()) {
if (!f.getName().startsWith(CHANGES_PREFIX)) {
continue;
}
String versionStr = f.getName().substring(CHANGES_PREFIX.length());
Integer v = Ints.tryParse(versionStr);
if (v == null || versionStr.length() != 4) {
log.warn("Unrecognized version in index directory: {}",
f.getAbsolutePath());
continue;
}
if (!versions.containsKey(v)) {
versions.put(v, new Version(null, v, true, getReady(cfg, v)));
}
}
return versions;
}
private void markNotReady(FileBasedConfig cfg, Iterable<Version> versions,
Collection<Version> inUse) {
boolean dirty = false;
for (Version v : versions) {
if (!inUse.contains(v) && v.exists) {
setReady(cfg, v.version, false);
dirty = true;
}
}
if (dirty) {
try {
cfg.save();
} catch (IOException e) {
throw fail(e);
}
}
}
private ProvisionException fail(Throwable t) {
ProvisionException e = new ProvisionException("Error scanning indexes");
e.initCause(t);
throw e;
}
@Override
public void stop() {
}
}

36
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
View File

@ -43,6 +43,8 @@ import com.google.gerrit.server.git.MultiProgressMonitor;
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeIndexer;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
@ -63,7 +65,6 @@ import com.google.inject.TypeLiteral;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
@ -98,14 +99,22 @@ public class Reindex extends SiteProgram {
@Option(name = "--threads", usage = "Number of threads to use for indexing")
private int threads = Runtime.getRuntime().availableProcessors();
@Option(name = "--schema-version",
usage = "Schema version to reindex; default is most recent version")
private Integer version;
@Option(name = "--output", usage = "Prefix for output; path for local disk index, or prefix for remote index")
private String outputBase;
@Option(name = "--verbose", usage = "Output debug information for each change")
private boolean verbose;
@Option(name = "--dry-run", usage = "Dry run: don't write anything to index")
private boolean dryRun;
private Injector dbInjector;
private Injector sysInjector;
private ChangeIndex index;
@Override
public int run() throws Exception {
@ -114,22 +123,22 @@ public class Reindex extends SiteProgram {
if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) {
throw die("index.type must be configured (or not SQL)");
}
if (version == null) {
version = ChangeSchemas.getLatest().getVersion();
}
LifecycleManager dbManager = new LifecycleManager();
dbManager.add(dbInjector);
dbManager.start();
sysInjector = createSysInjector();
// Delete before any index may be created depending on this data.
deleteAll();
LifecycleManager sysManager = new LifecycleManager();
sysManager.add(sysInjector);
sysManager.start();
index = sysInjector.getInstance(IndexCollection.class).getSearchIndex();
index.deleteAll();
int result = indexAll();
writeVersion();
index.markReady();
sysManager.stop();
dbManager.stop();
@ -142,7 +151,7 @@ public class Reindex extends SiteProgram {
AbstractModule changeIndexModule;
switch (IndexModule.getIndexType(dbInjector)) {
case LUCENE:
changeIndexModule = new LuceneIndexModule(false, threads, outputBase);
changeIndexModule = new LuceneIndexModule(version, threads, outputBase);
break;
case SOLR:
changeIndexModule = new SolrIndexModule(false, threads, outputBase);
@ -207,11 +216,6 @@ public class Reindex extends SiteProgram {
}
}
private void deleteAll() throws IOException {
ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
index.deleteAll();
}
private int indexAll() throws Exception {
ReviewDb db = sysInjector.getInstance(ReviewDb.class);
ListeningScheduledExecutorService executor = sysInjector.getInstance(
@ -464,10 +468,4 @@ public class Reindex extends SiteProgram {
}
}
}
private void writeVersion() throws IOException,
ConfigInvalidException {
ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
index.finishIndex();
}
}

25
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java
View File

@ -21,8 +21,6 @@ import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import org.eclipse.jgit.errors.ConfigInvalidException;
import java.io.IOException;
/**
@ -64,20 +62,27 @@ public interface ChangeIndex {
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p)
throws QueryParseException {
public ChangeDataSource getSource(Predicate<ChangeData> p) {
throw new UnsupportedOperationException();
}
@Override
public void finishIndex() {
public void close() {
// Do nothing.
}
@Override
public void markReady() {
throw new UnsupportedOperationException();
}
};
/** @return the schema version used by this index. */
public Schema<ChangeData> getSchema();
/** Close this index. */
public void close();
/**
* Insert a change document into the index.
* <p>
@ -137,11 +142,13 @@ public interface ChangeIndex {
throws QueryParseException;
/**
* Mark completion of indexing.
* Mark this index as up-to-date and ready to serve reads.
* <p>
* Should only be called immediately after a reindex, either during an online
* schema upgrade while actively writing to this index, or during an offline
* reindex.
*
* @throws ConfigInvalidException
* @throws IOException
*/
public void finishIndex() throws IOException,
ConfigInvalidException;
public void markReady() throws IOException;
}

17
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java
View File

@ -15,9 +15,9 @@
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.gerrit.server.query.change.ChangeData;
@ -58,23 +58,16 @@ public class ChangeSchemas {
return new Schema<ChangeData>(false, Arrays.asList(fields));
}
private static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
public static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
public Schema<ChangeData> get(int version) {
public static Schema<ChangeData> get(int version) {
Schema<ChangeData> schema = ALL.get(version);
checkArgument(schema != null, "Unrecognized schema version: %s", version);
return schema;
}
public static Schema<ChangeData> getLatestRelease() {
Schema<ChangeData> latest = null;
for (Schema<ChangeData> schema : ALL.values()) {
if (schema.isRelease()) {
latest = schema;
}
}
checkState(latest != null, "No released schema versions found");
return latest;
public static Schema<ChangeData> getLatest() {
return Iterables.getLast(ALL.values());
}
static {

20
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java
View File

@ -16,6 +16,7 @@ package com.google.gerrit.server.index;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@ -28,7 +29,7 @@ import javax.annotation.Nullable;
/** Dynamic pointers to the index versions used for searching and writing. */
@Singleton
public class IndexCollection {
public class IndexCollection implements LifecycleListener {
private final CopyOnWriteArrayList<ChangeIndex> writeIndexes;
private final AtomicReference<ChangeIndex> searchIndex;
@ -66,4 +67,21 @@ public class IndexCollection {
}
writeIndexes.add(index);
}
@Override
public void start() {
}
@Override
public void stop() {
ChangeIndex read = searchIndex.get();
if (read != null) {
read.close();
}
for (ChangeIndex write : writeIndexes) {
if (write != read) {
write.close();
}
}
}
}

6
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
View File

@ -16,11 +16,11 @@ package com.google.gerrit.server.index;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.git.WorkQueue.Executor;
import com.google.gerrit.server.query.change.ChangeQueryRewriter;
import com.google.inject.AbstractModule;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provides;
@ -34,7 +34,7 @@ import org.eclipse.jgit.lib.Config;
* This module should not be used directly except by specific secondary indexer
* implementations (e.g. Lucene).
*/
public class IndexModule extends AbstractModule {
public class IndexModule extends LifecycleModule {
public enum IndexType {
SQL, LUCENE, SOLR;
}
@ -57,6 +57,8 @@ public class IndexModule extends AbstractModule {
bind(ChangeIndexer.class).to(ChangeIndexerImpl.class);
bind(ChangeQueryRewriter.class).to(IndexRewriteImpl.class);
bind(IndexRewriteImpl.BasicRewritesImpl.class);
bind(IndexCollection.class);
listener().to(IndexCollection.class);
}
@Provides

6
gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java
View File

@ -90,7 +90,11 @@ public class IndexRewriteTest extends TestCase {
}
@Override
public void finishIndex() {
public void close() {
}
@Override
public void markReady() {
throw new UnsupportedOperationException();
}
}

4
gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java
View File

@ -31,8 +31,8 @@ import java.util.Map;
class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatest().getVersion(),
SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatest().getVersion());
public static File solrIndexConfig(SitePaths sitePaths) {
return new File(sitePaths.index_dir, "gerrit_index.config");

9
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
View File

@ -52,7 +52,6 @@ import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
@ -120,6 +119,11 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
return schema;
}
@Override
public void close() {
stop();
}
@Override
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
String id = cd.getId().toString();
@ -321,8 +325,7 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
}
@Override
public void finishIndex() throws IOException,
ConfigInvalidException {
public void markReady() throws IOException {
// TODO Move the schema version information to a special meta-document
FileBasedConfig cfg = new FileBasedConfig(
solrIndexConfig(sitePaths),

2
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java
View File

@ -61,6 +61,6 @@ public class SolrIndexModule extends LifecycleModule {
IndexCollection indexes,
FillArgs fillArgs) throws IOException {
return new SolrChangeIndex(cfg, fillArgs, sitePaths, indexes,
ChangeSchemas.getLatestRelease(), base);
ChangeSchemas.getLatest(), base);
}
}