opendev/gerrit
Code Issues Proposed changes

Merge changes from topic 'account-index-gh16'

Browse Source 
* changes:
  Encapsulate per-index utilities in a pair of generic classes
  Generify OnlineReindexer and friends
  Move Document generation to AbstractLuceneIndex
  Parameterize Lucene's QueryBuilder
  Extract SubIndex as an abstract Index implementation
  LuceneVersionManager: Pass "changes_" prefix as argument
  Extract a method for in-memory testing state in Lucene
  Move BooleanQuery static call into LuceneChangeModule
  Move GerritIndexWriterConfig to its own file
  Schema for secondary index over accounts
This commit is contained in:
Dave Borowitz
2016-03-18 11:02:37 +00:00
committed by Gerrit Code Review
parent 7c546d4133 886746ff6f
commit 0a0bb448f0
26 changed files with 1374 additions and 748 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gerrit-acceptance-framework/src/test/java/com/google/gerrit/acceptance/GerritServer.java
View File

@@ -26,7 +26,6 @@ import com.google.gerrit.pgm.Daemon;
import com.google.gerrit.pgm.Init;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.AsyncReceiveCommits;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.ssh.NoSshModule;
import com.google.gerrit.server.util.SocketUtil;
import com.google.gerrit.server.util.SystemLog;
@@ -142,8 +141,7 @@ public class GerritServer {
cfg.setBoolean("index", "lucene", "testInmemory", true);
cfg.setString("gitweb", null, "cgi", "");
daemon.setEnableHttpd(desc.httpd());
daemon.setLuceneModule(new LuceneIndexModule(
ChangeSchemas.getLatest().getVersion(), 0));
daemon.setLuceneModule(LuceneIndexModule.singleVersionAllLatest(0));
daemon.setDatabaseForTesting(ImmutableList.<Module>of(
new InMemoryTestingDatabaseModule(cfg)));
daemon.start();

137
gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java → gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
View File

@@ -20,9 +20,22 @@ import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gerrit.lucene.LuceneChangeIndex.GerritIndexWriterConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TrackingIndexWriter;
@@ -33,12 +46,13 @@ import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
@@ -46,25 +60,45 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/** Piece of the change index that is implemented as a separate Lucene index. */
public class SubIndex {
private static final Logger log = LoggerFactory.getLogger(SubIndex.class);
/** Basic Lucene index implementation. */
public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
private static final Logger log =
LoggerFactory.getLogger(AbstractLuceneIndex.class);
static String sortFieldName(FieldDef<?, ?> f) {
return f.getName() + "_SORT";
}
public static void setReady(SitePaths sitePaths, int version, boolean ready)
throws IOException {
try {
// TODO(dborowitz): Totally broken for non-change indexes.
FileBasedConfig cfg =
LuceneVersionManager.loadGerritIndexConfig(sitePaths);
LuceneVersionManager.setReady(cfg, version, ready);
cfg.save();
} catch (ConfigInvalidException e) {
throw new IOException(e);
}
}
private final Schema<V> schema;
private final SitePaths sitePaths;
private final Directory dir;
private final TrackingIndexWriter writer;
private final ReferenceManager<IndexSearcher> searcherManager;
private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
private final Set<NrtFuture> notDoneNrtFutures;
SubIndex(Path path, GerritIndexWriterConfig writerConfig,
SearcherFactory searcherFactory) throws IOException {
this(FSDirectory.open(path), path.getFileName().toString(), writerConfig,
searcherFactory);
}
SubIndex(Directory dir, final String dirName,
AbstractLuceneIndex(
Schema<V> schema,
SitePaths sitePaths,
Directory dir,
final String name,
GerritIndexWriterConfig writerConfig,
SearcherFactory searcherFactory) throws IOException {
this.schema = schema;
this.sitePaths = sitePaths;
this.dir = dir;
IndexWriter delegateWriter;
long commitPeriod = writerConfig.getCommitWithinMs();
@@ -80,7 +114,7 @@ public class SubIndex {
delegateWriter = autoCommitWriter;
new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder()
.setNameFormat("Commit-%d " + dirName)
.setNameFormat("Commit-%d " + name)
.setDaemon(true)
.build())
.scheduleAtFixedRate(new Runnable() {
@@ -92,14 +126,14 @@ public class SubIndex {
autoCommitWriter.commit();
}
} catch (IOException e) {
log.error("Error committing Lucene index " + dirName, e);
log.error("Error committing " + name + " Lucene index", e);
} catch (OutOfMemoryError e) {
log.error("Error committing Lucene index " + dirName, e);
log.error("Error committing " + name + " Lucene index", e);
try {
autoCommitWriter.close();
} catch (IOException e2) {
log.error("SEVERE: Error closing Lucene index " + dirName
+ " after OOM; index may be corrupted.", e);
log.error("SEVERE: Error closing " + name
+ " Lucene index after OOM; index may be corrupted.", e);
}
}
}
@@ -115,7 +149,7 @@ public class SubIndex {
writer, searcherManager,
0.500 /* maximum stale age (seconds) */,
0.010 /* minimum stale age (seconds) */);
reopenThread.setName("NRT " + dirName);
reopenThread.setName("NRT " + name);
reopenThread.setPriority(Math.min(
Thread.currentThread().getPriority() + 2,
Thread.MAX_PRIORITY));
@@ -145,7 +179,13 @@ public class SubIndex {
reopenThread.start();
}
void close() {
@Override
public void markReady(boolean ready) throws IOException {
setReady(sitePaths, schema.getVersion(), ready);
}
@Override
public void close() {
reopenThread.close();
// Closing the reopen thread sets its generation to Long.MAX_VALUE, but we
@@ -187,7 +227,8 @@ public class SubIndex {
return new NrtFuture(writer.deleteDocuments(term));
}
void deleteAll() throws IOException {
@Override
public void deleteAll() throws IOException {
writer.deleteAll();
}
@@ -203,6 +244,55 @@ public class SubIndex {
searcherManager.release(searcher);
}
Document toDocument(V obj, FillArgs fillArgs) {
Document result = new Document();
for (Values<V> vs : schema.buildFields(obj, fillArgs)) {
if (vs.getValues() != null) {
add(result, vs);
}
}
return result;
}
void add(Document doc, Values<V> values) {
String name = values.getField().getName();
FieldType<?> type = values.getField().getType();
Store store = store(values.getField());
if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
for (Object value : values.getValues()) {
doc.add(new IntField(name, (Integer) value, store));
}
} else if (type == FieldType.LONG) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, (Long) value, store));
}
} else if (type == FieldType.TIMESTAMP) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX) {
for (Object value : values.getValues()) {
doc.add(new StringField(name, (String) value, store));
}
} else if (type == FieldType.FULL_TEXT) {
for (Object value : values.getValues()) {
doc.add(new TextField(name, (String) value, store));
}
} else if (type == FieldType.STORED_ONLY) {
for (Object value : values.getValues()) {
doc.add(new StoredField(name, (byte[]) value));
}
} else {
throw FieldType.badFieldType(type);
}
}
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
private final class NrtFuture extends AbstractFuture<Void> {
private final long gen;
@@ -287,4 +377,9 @@ public class SubIndex {
}
}
}
@Override
public Schema<V> getSchema() {
return schema;
}
}

98
gerrit-lucene/src/main/java/com/google/gerrit/lucene/ChangeSubIndex.java Normal file
View File

@@ -0,0 +1,98 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.gerrit.lucene.LuceneChangeIndex.ID_SORT_FIELD;
import static com.google.gerrit.lucene.LuceneChangeIndex.UPDATED_SORT_FIELD;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.query.DataSource;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Timestamp;
public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData>
implements ChangeIndex {
ChangeSubIndex(
Schema<ChangeData> schema,
SitePaths sitePaths,
Path path,
GerritIndexWriterConfig writerConfig,
SearcherFactory searcherFactory) throws IOException {
this(schema, sitePaths, FSDirectory.open(path),
path.getFileName().toString(), writerConfig, searcherFactory);
}
ChangeSubIndex(
Schema<ChangeData> schema,
SitePaths sitePaths,
Directory dir,
String name,
GerritIndexWriterConfig writerConfig,
SearcherFactory searcherFactory) throws IOException {
super(schema, sitePaths, dir, name, writerConfig, searcherFactory);
}
@Override
public void replace(ChangeData obj) throws IOException {
throw new UnsupportedOperationException(
"don't use ChangeSubIndex directly");
}
@Override
public void delete(Change.Id key) throws IOException {
throw new UnsupportedOperationException(
"don't use ChangeSubIndex directly");
}
@Override
public DataSource<ChangeData> getSource(Predicate<ChangeData> p,
QueryOptions opts) throws QueryParseException {
throw new UnsupportedOperationException(
"don't use ChangeSubIndex directly");
}
@Override
void add(Document doc, Values<ChangeData> values) {
// Add separate DocValues fields for those fields needed for sorting.
FieldDef<ChangeData, ?> f = values.getField();
if (f == ChangeField.LEGACY_ID) {
int v = (Integer) getOnlyElement(values.getValues());
doc.add(new NumericDocValuesField(ID_SORT_FIELD, v));
} else if (f == ChangeField.UPDATED) {
long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
}
super.add(doc, values);
}
}

77
gerrit-lucene/src/main/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java Normal file
View File

@@ -0,0 +1,77 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.server.config.ConfigUtil;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.eclipse.jgit.lib.Config;
import java.util.Map;
/**
* Combination of Lucene {@link IndexWriterConfig} with additional
* Gerrit-specific options.
*/
class GerritIndexWriterConfig {
private static final Map<String, String> CUSTOM_CHAR_MAPPING =
ImmutableMap.of("_", " ", ".", " ");
private final IndexWriterConfig luceneConfig;
private long commitWithinMs;
private final CustomMappingAnalyzer analyzer;
GerritIndexWriterConfig(Config cfg, String name) {
analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(
CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
luceneConfig = new IndexWriterConfig(analyzer)
.setOpenMode(OpenMode.CREATE_OR_APPEND)
.setCommitOnClose(true);
double m = 1 << 20;
luceneConfig.setRAMBufferSizeMB(cfg.getLong(
"index", name, "ramBufferSize",
(long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m);
luceneConfig.setMaxBufferedDocs(cfg.getInt(
"index", name, "maxBufferedDocs",
IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS));
try {
commitWithinMs =
ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin",
MILLISECONDS.convert(5, MINUTES), MILLISECONDS);
} catch (IllegalArgumentException e) {
commitWithinMs = cfg.getLong("index", name, "commitWithin", 0);
}
}
CustomMappingAnalyzer getAnalyzer() {
return analyzer;
}
IndexWriterConfig getLuceneConfig() {
return luceneConfig;
}
long getCommitWithinMs() {
return commitWithinMs;
}
}

215
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -15,17 +15,15 @@
package com.google.gerrit.lucene;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.gerrit.lucene.AbstractLuceneIndex.sortFieldName;
import static com.google.gerrit.lucene.LuceneVersionManager.CHANGES_PREFIX;
import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
import static com.google.gerrit.server.index.change.IndexRewriter.CLOSED_STATUSES;
import static com.google.gerrit.server.index.change.IndexRewriter.OPEN_STATUSES;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -37,16 +35,12 @@ import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeField.ChangeProtoField;
import com.google.gerrit.server.index.change.ChangeField.PatchSetApprovalProtoField;
@@ -64,23 +58,10 @@ import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@@ -92,20 +73,16 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@@ -124,6 +101,11 @@ public class LuceneChangeIndex implements ChangeIndex {
public static final String CHANGES_OPEN = "open";
public static final String CHANGES_CLOSED = "closed";
static final String UPDATED_SORT_FIELD =
sortFieldName(ChangeField.UPDATED);
static final String ID_SORT_FIELD =
sortFieldName(ChangeField.LEGACY_ID);
private static final String ADDED_FIELD = ChangeField.ADDED.getName();
private static final String APPROVAL_FIELD = ChangeField.APPROVAL.getName();
private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
@@ -132,68 +114,13 @@ public class LuceneChangeIndex implements ChangeIndex {
private static final String PATCH_SET_FIELD = ChangeField.PATCH_SET.getName();
private static final String REVIEWEDBY_FIELD =
ChangeField.REVIEWEDBY.getName();
private static final String UPDATED_SORT_FIELD =
sortFieldName(ChangeField.UPDATED);
private static final String ID_SORT_FIELD =
sortFieldName(ChangeField.LEGACY_ID);
private static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
"_", " ", ".", " ");
public static void setReady(SitePaths sitePaths, int version, boolean ready)
throws IOException {
try {
FileBasedConfig cfg =
LuceneVersionManager.loadGerritIndexConfig(sitePaths);
LuceneVersionManager.setReady(cfg, version, ready);
cfg.save();
} catch (ConfigInvalidException e) {
throw new IOException(e);
}
static Term idTerm(ChangeData cd) {
return QueryBuilder.intTerm(LEGACY_ID.getName(), cd.getId().get());
}
private static String sortFieldName(FieldDef<?, ?> f) {
return f.getName() + "_SORT";
}
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema);
}
static class GerritIndexWriterConfig {
private final IndexWriterConfig luceneConfig;
private long commitWithinMs;
private GerritIndexWriterConfig(Config cfg, String name) {
CustomMappingAnalyzer analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(
CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
luceneConfig = new IndexWriterConfig(analyzer)
.setOpenMode(OpenMode.CREATE_OR_APPEND)
.setCommitOnClose(true);
double m = 1 << 20;
luceneConfig.setRAMBufferSizeMB(cfg.getLong(
"index", name, "ramBufferSize",
(long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m);
luceneConfig.setMaxBufferedDocs(cfg.getInt(
"index", name, "maxBufferedDocs",
IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS));
try {
commitWithinMs =
ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin",
MILLISECONDS.convert(5, MINUTES), MILLISECONDS);
} catch (IllegalArgumentException e) {
commitWithinMs = cfg.getLong("index", name, "commitWithin", 0);
}
}
IndexWriterConfig getLuceneConfig() {
return luceneConfig;
}
long getCommitWithinMs() {
return commitWithinMs;
}
static Term idTerm(Change.Id id) {
return QueryBuilder.intTerm(LEGACY_ID.getName(), id.get());
}
private final SitePaths sitePaths;
@@ -202,9 +129,9 @@ public class LuceneChangeIndex implements ChangeIndex {
private final Provider<ReviewDb> db;
private final ChangeData.Factory changeDataFactory;
private final Schema<ChangeData> schema;
private final QueryBuilder queryBuilder;
private final SubIndex openIndex;
private final SubIndex closedIndex;
private final QueryBuilder<ChangeData> queryBuilder;
private final ChangeSubIndex openIndex;
private final ChangeSubIndex closedIndex;
@AssistedInject
LuceneChangeIndex(
@@ -222,31 +149,25 @@ public class LuceneChangeIndex implements ChangeIndex {
this.changeDataFactory = changeDataFactory;
this.schema = schema;
CustomMappingAnalyzer analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET),
CUSTOM_CHAR_MAPPING);
queryBuilder = new QueryBuilder(analyzer);
BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms",
BooleanQuery.getMaxClauseCount()));
GerritIndexWriterConfig openConfig =
new GerritIndexWriterConfig(cfg, "changes_open");
GerritIndexWriterConfig closedConfig =
new GerritIndexWriterConfig(cfg, "changes_closed");
queryBuilder = new QueryBuilder<>(schema, openConfig.getAnalyzer());
SearcherFactory searcherFactory = new SearcherFactory();
if (cfg.getBoolean("index", "lucene", "testInmemory", false)) {
openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig,
searcherFactory);
closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig,
searcherFactory);
if (LuceneIndexModule.isInMemoryTest(cfg)) {
openIndex = new ChangeSubIndex(schema, sitePaths, new RAMDirectory(),
"ramOpen", openConfig, searcherFactory);
closedIndex = new ChangeSubIndex(schema, sitePaths, new RAMDirectory(),
"ramClosed", closedConfig, searcherFactory);
} else {
Path dir = LuceneVersionManager.getDir(sitePaths, schema);
openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig,
searcherFactory);
closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig,
searcherFactory);
Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES_PREFIX, schema);
openIndex = new ChangeSubIndex(schema, sitePaths,
dir.resolve(CHANGES_OPEN), openConfig, searcherFactory);
closedIndex = new ChangeSubIndex(schema, sitePaths,
dir.resolve(CHANGES_CLOSED), closedConfig, searcherFactory);
}
}
@@ -275,8 +196,10 @@ public class LuceneChangeIndex implements ChangeIndex {
@Override
public void replace(ChangeData cd) throws IOException {
Term id = QueryBuilder.idTerm(cd);
Document doc = toDocument(cd);
Term id = LuceneChangeIndex.idTerm(cd);
// toDocument is essentially static and doesn't depend on the specific
// sub-index, so just pick one.
Document doc = openIndex.toDocument(cd, fillArgs);
try {
if (cd.change().getStatus().isOpen()) {
Futures.allAsList(
@@ -294,7 +217,7 @@ public class LuceneChangeIndex implements ChangeIndex {
@Override
public void delete(Change.Id id) throws IOException {
Term idTerm = QueryBuilder.idTerm(id);
Term idTerm = LuceneChangeIndex.idTerm(id);
try {
Futures.allAsList(
openIndex.delete(idTerm),
@@ -314,7 +237,7 @@ public class LuceneChangeIndex implements ChangeIndex {
public ChangeDataSource getSource(Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
Set<Change.Status> statuses = IndexRewriter.getPossibleStatus(p);
List<SubIndex> indexes = Lists.newArrayListWithCapacity(2);
List<ChangeSubIndex> indexes = Lists.newArrayListWithCapacity(2);
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(openIndex);
}
@@ -326,7 +249,9 @@ public class LuceneChangeIndex implements ChangeIndex {
@Override
public void markReady(boolean ready) throws IOException {
setReady(sitePaths, schema.getVersion(), ready);
// Do not delegate to ChangeSubIndex#markReady, since changes have an
// additional level of directory nesting.
AbstractLuceneIndex.setReady(sitePaths, schema.getVersion(), ready);
}
private Sort getSort() {
@@ -335,21 +260,17 @@ public class LuceneChangeIndex implements ChangeIndex {
new SortField(ID_SORT_FIELD, SortField.Type.LONG, true));
}
public SubIndex getOpenChangesIndex() {
return openIndex;
}
public SubIndex getClosedChangesIndex() {
public ChangeSubIndex getClosedChangesIndex() {
return closedIndex;
}
private class QuerySource implements ChangeDataSource {
private final List<SubIndex> indexes;
private final List<ChangeSubIndex> indexes;
private final Query query;
private final QueryOptions opts;
private final Sort sort;
private QuerySource(List<SubIndex> indexes, Query query, QueryOptions opts,
private QuerySource(List<ChangeSubIndex> indexes, Query query, QueryOptions opts,
Sort sort) {
this.indexes = indexes;
this.query = checkNotNull(query, "null query from Lucene");
@@ -557,64 +478,4 @@ public class LuceneChangeIndex implements ChangeIndex {
}
return result;
}
private Document toDocument(ChangeData cd) {
Document result = new Document();
for (Values<ChangeData> vs : schema.buildFields(cd, fillArgs)) {
if (vs.getValues() != null) {
add(result, vs);
}
}
return result;
}
private void add(Document doc, Values<ChangeData> values) {
String name = values.getField().getName();
FieldType<?> type = values.getField().getType();
Store store = store(values.getField());
FieldDef<ChangeData, ?> f = values.getField();
// Add separate DocValues fields for those fields needed for sorting.
if (f == ChangeField.LEGACY_ID) {
int v = (Integer) getOnlyElement(values.getValues());
doc.add(new NumericDocValuesField(sortFieldName(f), v));
} else if (f == ChangeField.UPDATED) {
long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
}
if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
for (Object value : values.getValues()) {
doc.add(new IntField(name, (Integer) value, store));
}
} else if (type == FieldType.LONG) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, (Long) value, store));
}
} else if (type == FieldType.TIMESTAMP) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX) {
for (Object value : values.getValues()) {
doc.add(new StringField(name, (String) value, store));
}
} else if (type == FieldType.FULL_TEXT) {
for (Object value : values.getValues()) {
doc.add(new TextField(name, (String) value, store));
}
} else if (type == FieldType.STORED_ONLY) {
for (Object value : values.getValues()) {
doc.add(new StoredField(name, (byte[]) value));
}
} else {
throw FieldType.badFieldType(type);
}
}
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
}

105
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
View File

@@ -14,40 +14,69 @@
package com.google.gerrit.lucene;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexDefinition;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.name.Named;
import com.google.inject.name.Names;
import org.apache.lucene.search.BooleanQuery;
import org.eclipse.jgit.lib.Config;
public class LuceneIndexModule extends LifecycleModule {
private final Integer singleVersion;
private final int threads;
import java.util.Collection;
import java.util.Map;
public LuceneIndexModule() {
this(null, 0);
public class LuceneIndexModule extends LifecycleModule {
private static final String SINGLE_VERSIONS =
"LuceneIndexModule/SingleVersions";
public static LuceneIndexModule singleVersionAllLatest(int threads) {
return new LuceneIndexModule(ImmutableMap.<String, Integer> of(), threads);
}
public LuceneIndexModule(Integer singleVersion, int threads) {
this.singleVersion = singleVersion;
public static LuceneIndexModule singleVersionWithExplicitVersions(
Map<String, Integer> versions, int threads) {
return new LuceneIndexModule(versions, threads);
}
public static LuceneIndexModule latestVersionWithOnlineUpgrade() {
return new LuceneIndexModule(null, 0);
}
static boolean isInMemoryTest(Config cfg) {
return cfg.getBoolean("index", "lucene", "testInmemory", false);
}
private final int threads;
private final Map<String, Integer> singleVersions;
private LuceneIndexModule(Map<String, Integer> singleVersions, int threads) {
this.singleVersions = singleVersions;
this.threads = threads;
}
@Override
protected void configure() {
factory(LuceneChangeIndex.Factory.class);
factory(OnlineReindexer.Factory.class);
install(
new FactoryModuleBuilder()
.implement(ChangeIndex.class, LuceneChangeIndex.class)
.build(ChangeIndex.Factory.class));
install(new IndexModule(threads));
if (singleVersion == null) {
if (singleVersions == null) {
install(new MultiVersionModule());
} else {
install(new SingleVersionModule());
@@ -57,6 +86,8 @@ public class LuceneIndexModule extends LifecycleModule {
@Provides
@Singleton
IndexConfig getIndexConfig(@GerritServerConfig Config cfg) {
BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms",
BooleanQuery.getMaxClauseCount()));
return IndexConfig.fromConfig(cfg);
}
@@ -71,34 +102,48 @@ public class LuceneIndexModule extends LifecycleModule {
@Override
public void configure() {
listener().to(SingleVersionListener.class);
}
@Provides
@Singleton
LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory) {
Schema<ChangeData> schema = singleVersion != null
? ChangeSchemas.get(singleVersion)
: ChangeSchemas.getLatest();
return factory.create(schema);
bind(new TypeLiteral<Map<String, Integer>>() {})
.annotatedWith(Names.named(SINGLE_VERSIONS))
.toInstance(singleVersions);
}
}
@Singleton
static class SingleVersionListener implements LifecycleListener {
private final ChangeIndexCollection indexes;
private final LuceneChangeIndex index;
private final Collection<IndexDefinition<?, ?, ?>> defs;
private final Map<String, Integer> singleVersions;
@Inject
SingleVersionListener(ChangeIndexCollection indexes,
LuceneChangeIndex index) {
this.indexes = indexes;
this.index = index;
SingleVersionListener(
Collection<IndexDefinition<?, ?, ?>> defs,
@Named(SINGLE_VERSIONS) Map<String, Integer> singleVersions) {
this.defs = defs;
this.singleVersions = singleVersions;
}
@Override
public void start() {
indexes.setSearchIndex(index);
indexes.addWriteIndex(index);
for (IndexDefinition<?, ?, ?> def : defs) {
start(def);
}
}
private <K, V, I extends Index<K, V>> void start(
IndexDefinition<K, V, I> def) {
Schema<V> schema;
Integer v = singleVersions.get(def.getName());
if (v == null) {
schema = def.getLatest();
} else {
schema = def.getSchemas().get(v);
if (schema == null) {
throw new ProvisionException(String.format(
"Unrecognized %s schema version: %s", def.getName(), v));
}
}
I index = def.getIndexFactory().create(schema);
def.getIndexCollection().setSearchIndex(index);
def.getIndexCollection().addWriteIndex(index);
}
@Override

28
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
View File

@@ -20,11 +20,14 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.OnlineReindexer;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.index.change.ChangeIndexDefintion;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
@@ -50,7 +53,7 @@ public class LuceneVersionManager implements LifecycleListener {
private static final Logger log = LoggerFactory
.getLogger(LuceneVersionManager.class);
private static final String CHANGES_PREFIX = "changes_";
static final String CHANGES_PREFIX = "changes_";
private static class Version {
private final Schema<ChangeData> schema;
@@ -68,9 +71,9 @@ public class LuceneVersionManager implements LifecycleListener {
}
}
static Path getDir(SitePaths sitePaths, Schema<ChangeData> schema) {
static Path getDir(SitePaths sitePaths, String prefix, Schema<?> schema) {
return sitePaths.index_dir.resolve(String.format("%s%04d",
CHANGES_PREFIX, schema.getVersion()));
prefix, schema.getVersion()));
}
static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths)
@@ -93,9 +96,9 @@ public class LuceneVersionManager implements LifecycleListener {
private final SitePaths sitePaths;
private final LuceneChangeIndex.Factory indexFactory;
private final ChangeIndexCollection indexes;
private final OnlineReindexer.Factory reindexerFactory;
private final ChangeIndexDefintion changeDef;
private final boolean onlineUpgrade;
private OnlineReindexer reindexer;
private OnlineReindexer<Change.Id, ChangeData, ChangeIndex> reindexer;
@Inject
LuceneVersionManager(
@@ -103,11 +106,11 @@ public class LuceneVersionManager implements LifecycleListener {
SitePaths sitePaths,
LuceneChangeIndex.Factory indexFactory,
ChangeIndexCollection indexes,
OnlineReindexer.Factory reindexerFactory) {
ChangeIndexDefintion changeDef) {
this.sitePaths = sitePaths;
this.indexFactory = indexFactory;
this.indexes = indexes;
this.reindexerFactory = reindexerFactory;
this.changeDef = changeDef;
this.onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true);
}
@@ -157,7 +160,8 @@ public class LuceneVersionManager implements LifecycleListener {
}
markNotReady(cfg, versions.values(), write);
LuceneChangeIndex searchIndex = indexFactory.create(search.schema);
LuceneChangeIndex searchIndex =
(LuceneChangeIndex) indexFactory.create(search.schema);
indexes.setSearchIndex(searchIndex);
for (Version v : write) {
if (v.schema != null) {
@@ -171,7 +175,7 @@ public class LuceneVersionManager implements LifecycleListener {
int latest = write.get(0).version;
if (onlineUpgrade && latest != search.version) {
reindexer = reindexerFactory.create(latest);
reindexer = new OnlineReindexer<>(changeDef, latest);
reindexer.start();
}
}
@@ -223,8 +227,8 @@ public class LuceneVersionManager implements LifecycleListener {
private TreeMap<Integer, Version> scanVersions(Config cfg) {
TreeMap<Integer, Version> versions = Maps.newTreeMap();
for (Schema<ChangeData> schema : ChangeSchemas.ALL.values()) {
Path p = getDir(sitePaths, schema);
for (Schema<ChangeData> schema : changeDef.getSchemas().values()) {
Path p = getDir(sitePaths, CHANGES_PREFIX, schema);
boolean isDir = Files.isDirectory(p);
if (Files.exists(p) && !isDir) {
log.warn("Not a directory: %s", p.toAbsolutePath());

77
gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java
View File

@@ -14,24 +14,23 @@
package com.google.gerrit.lucene;
import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
import com.google.common.collect.Lists;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.IntegerRangePredicate;
import com.google.gerrit.server.index.RegexPredicate;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.query.AndPredicate;
import com.google.gerrit.server.query.NotPredicate;
import com.google.gerrit.server.query.OrPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
@@ -48,23 +47,22 @@ import org.apache.lucene.util.NumericUtils;
import java.util.Date;
import java.util.List;
public class QueryBuilder {
public static Term idTerm(ChangeData cd) {
return intTerm(LEGACY_ID.getName(), cd.getId().get());
}
public static Term idTerm(Change.Id id) {
return intTerm(LEGACY_ID.getName(), id.get());
public class QueryBuilder<V> {
static Term intTerm(String name, int value) {
BytesRefBuilder builder = new BytesRefBuilder();
NumericUtils.intToPrefixCodedBytes(value, 0, builder);
return new Term(name, builder.get());
}
private final Schema<V> schema;
private final org.apache.lucene.util.QueryBuilder queryBuilder;
public QueryBuilder(Analyzer analyzer) {
public QueryBuilder(Schema<V> schema, Analyzer analyzer) {
this.schema = schema;
queryBuilder = new org.apache.lucene.util.QueryBuilder(analyzer);
}
public Query toQuery(Predicate<ChangeData> p) throws QueryParseException {
public Query toQuery(Predicate<V> p) throws QueryParseException {
if (p instanceof AndPredicate) {
return and(p);
} else if (p instanceof OrPredicate) {
@@ -72,13 +70,13 @@ public class QueryBuilder {
} else if (p instanceof NotPredicate) {
return not(p);
} else if (p instanceof IndexPredicate) {
return fieldQuery((IndexPredicate<ChangeData>) p);
return fieldQuery((IndexPredicate<V>) p);
} else {
throw new QueryParseException("cannot create query for index: " + p);
}
}
private Query or(Predicate<ChangeData> p)
private Query or(Predicate<V> p)
throws QueryParseException {
try {
BooleanQuery.Builder q = new BooleanQuery.Builder();
@@ -91,17 +89,17 @@ public class QueryBuilder {
}
}
private Query and(Predicate<ChangeData> p)
private Query and(Predicate<V> p)
throws QueryParseException {
try {
BooleanQuery.Builder b = new BooleanQuery.Builder();
List<Query> not = Lists.newArrayListWithCapacity(p.getChildCount());
for (int i = 0; i < p.getChildCount(); i++) {
Predicate<ChangeData> c = p.getChild(i);
Predicate<V> c = p.getChild(i);
if (c instanceof NotPredicate) {
Predicate<ChangeData> n = c.getChild(0);
Predicate<V> n = c.getChild(0);
if (n instanceof TimestampRangePredicate) {
b.add(notTimestamp((TimestampRangePredicate<ChangeData>) n), MUST);
b.add(notTimestamp((TimestampRangePredicate<V>) n), MUST);
} else {
not.add(toQuery(n));
}
@@ -118,11 +116,11 @@ public class QueryBuilder {
}
}
private Query not(Predicate<ChangeData> p)
private Query not(Predicate<V> p)
throws QueryParseException {
Predicate<ChangeData> n = p.getChild(0);
Predicate<V> n = p.getChild(0);
if (n instanceof TimestampRangePredicate) {
return notTimestamp((TimestampRangePredicate<ChangeData>) n);
return notTimestamp((TimestampRangePredicate<V>) n);
}
// Lucene does not support negation, start with all and subtract.
@@ -132,8 +130,11 @@ public class QueryBuilder {
.build();
}
private Query fieldQuery(IndexPredicate<ChangeData> p)
private Query fieldQuery(IndexPredicate<V> p)
throws QueryParseException {
checkArgument(schema.hasField(p.getField()),
"field not in schema v%s: %s", schema.getVersion(),
p.getField().getName());
if (p.getType() == FieldType.INTEGER) {
return intQuery(p);
} else if (p.getType() == FieldType.INTEGER_RANGE) {
@@ -151,13 +152,7 @@ public class QueryBuilder {
}
}
private static Term intTerm(String name, int value) {
BytesRefBuilder builder = new BytesRefBuilder();
NumericUtils.intToPrefixCodedBytes(value, 0, builder);
return new Term(name, builder.get());
}
private Query intQuery(IndexPredicate<ChangeData> p)
private Query intQuery(IndexPredicate<V> p)
throws QueryParseException {
int value;
try {
@@ -170,11 +165,11 @@ public class QueryBuilder {
return new TermQuery(intTerm(p.getField().getName(), value));
}
private Query intRangeQuery(IndexPredicate<ChangeData> p)
private Query intRangeQuery(IndexPredicate<V> p)
throws QueryParseException {
if (p instanceof IntegerRangePredicate) {
IntegerRangePredicate<ChangeData> r =
(IntegerRangePredicate<ChangeData>) p;
IntegerRangePredicate<V> r =
(IntegerRangePredicate<V>) p;
int minimum = r.getMinimumValue();
int maximum = r.getMaximumValue();
if (minimum == maximum) {
@@ -192,11 +187,11 @@ public class QueryBuilder {
throw new QueryParseException("not an integer range: " + p);
}
private Query timestampQuery(IndexPredicate<ChangeData> p)
private Query timestampQuery(IndexPredicate<V> p)
throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<ChangeData> r =
(TimestampRangePredicate<ChangeData>) p;
TimestampRangePredicate<V> r =
(TimestampRangePredicate<V>) p;
return NumericRangeQuery.newLongRange(
r.getField().getName(),
r.getMinTimestamp().getTime(),
@@ -206,7 +201,7 @@ public class QueryBuilder {
throw new QueryParseException("not a timestamp: " + p);
}
private Query notTimestamp(TimestampRangePredicate<ChangeData> r)
private Query notTimestamp(TimestampRangePredicate<V> r)
throws QueryParseException {
if (r.getMinTimestamp().getTime() == 0) {
return NumericRangeQuery.newLongRange(
@@ -218,7 +213,7 @@ public class QueryBuilder {
throw new QueryParseException("cannot negate: " + r);
}
private Query exactQuery(IndexPredicate<ChangeData> p) {
private Query exactQuery(IndexPredicate<V> p) {
if (p instanceof RegexPredicate<?>) {
return regexQuery(p);
} else {
@@ -226,7 +221,7 @@ public class QueryBuilder {
}
}
private Query regexQuery(IndexPredicate<ChangeData> p) {
private Query regexQuery(IndexPredicate<V> p) {
String re = p.getValue();
if (re.startsWith("^")) {
re = re.substring(1);
@@ -237,11 +232,11 @@ public class QueryBuilder {
return new RegexpQuery(new Term(p.getField().getName(), re));
}
private Query prefixQuery(IndexPredicate<ChangeData> p) {
private Query prefixQuery(IndexPredicate<V> p) {
return new PrefixQuery(new Term(p.getField().getName(), p.getValue()));
}
private Query fullTextQuery(IndexPredicate<ChangeData> p)
private Query fullTextQuery(IndexPredicate<V> p)
throws QueryParseException {
String value = p.getValue();
if (value == null) {

4
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java
View File

@@ -403,7 +403,9 @@ public class Daemon extends SiteProgram {
}
switch (indexType) {
case LUCENE:
return luceneModule != null ? luceneModule : new LuceneIndexModule();
return luceneModule != null
? luceneModule
: LuceneIndexModule.latestVersionWithOnlineUpgrade();
default:
throw new IllegalStateException("unsupported index.type = " + indexType);
}

27
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
View File

@@ -31,9 +31,10 @@ import com.google.gerrit.server.git.ScanningChangeCacheImpl;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.SiteIndexer;
import com.google.gerrit.server.index.change.AllChangesIndexer;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Injector;
@@ -47,7 +48,9 @@ import org.eclipse.jgit.lib.TextProgressMonitor;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.kohsuke.args4j.Option;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -55,9 +58,9 @@ public class Reindex extends SiteProgram {
@Option(name = "--threads", usage = "Number of threads to use for indexing")
private int threads = Runtime.getRuntime().availableProcessors();
@Option(name = "--schema-version",
usage = "Schema version to reindex; default is most recent version")
private Integer version;
@Option(name = "--changes-schema-version",
usage = "Schema version to reindex, for changes; default is most recent version")
private Integer changesVersion;
@Option(name = "--verbose", usage = "Output debug information for each change")
private boolean verbose;
@@ -82,9 +85,6 @@ public class Reindex extends SiteProgram {
checkNotSlaveMode();
disableLuceneAutomaticCommit();
disableChangeCache();
if (version == null) {
version = ChangeSchemas.getLatest().getVersion();
}
LifecycleManager dbManager = new LifecycleManager();
dbManager.add(dbInjector);
dbManager.start();
@@ -120,11 +120,16 @@ public class Reindex extends SiteProgram {
}
private Injector createSysInjector() {
Map<String, Integer> versions = new HashMap<>();
if (changesVersion != null) {
versions.put(ChangeSchemaDefinitions.INSTANCE.getName(), changesVersion);
}
List<Module> modules = Lists.newArrayList();
Module changeIndexModule;
switch (IndexModule.getIndexType(dbInjector)) {
case LUCENE:
changeIndexModule = new LuceneIndexModule(version, threads);
changeIndexModule = LuceneIndexModule.singleVersionWithExplicitVersions(
versions, threads);
break;
default:
throw new IllegalStateException("unsupported index.type");
@@ -164,9 +169,9 @@ public class Reindex extends SiteProgram {
}
pm.endTask();
SiteIndexer batchIndexer =
sysInjector.getInstance(SiteIndexer.class);
SiteIndexer.Result result = batchIndexer.setNumChanges(changeCount)
AllChangesIndexer batchIndexer =
sysInjector.getInstance(AllChangesIndexer.class);
SiteIndexer.Result result = batchIndexer.setTotalWork(changeCount)
.setProgressOut(System.err)
.setVerboseOut(verbose ? System.out : NullOutputStream.INSTANCE)
.indexAll(index, projects);

12
gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java
View File

@@ -15,14 +15,15 @@
package com.google.gerrit.pgm.init;
import com.google.common.collect.Iterables;
import com.google.gerrit.lucene.LuceneChangeIndex;
import com.google.gerrit.lucene.AbstractLuceneIndex;
import com.google.gerrit.pgm.init.api.ConsoleUI;
import com.google.gerrit.pgm.init.api.InitFlags;
import com.google.gerrit.pgm.init.api.InitStep;
import com.google.gerrit.pgm.init.api.Section;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.index.SchemaDefinitions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -57,8 +58,11 @@ class InitIndex implements InitStep {
ui.header("Index");
IndexType type = index.select("Type", "type", IndexType.LUCENE);
LuceneChangeIndex.setReady(
site, ChangeSchemas.getLatest().getVersion(), true);
for (SchemaDefinitions<?> def : IndexModule.ALL_SCHEMA_DEFS) {
// TODO(dborowitz): Totally broken for non-change indexes.
AbstractLuceneIndex.setReady(
site, def.getLatest().getVersion(), true);
}
if ((site.isNew || isEmptySite()) && type == IndexType.LUCENE) {
} else {
final String message = String.format(

9
gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java
View File

@@ -14,15 +14,24 @@
package com.google.gerrit.server.index;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.DummyChangeIndex;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.AbstractModule;
public class DummyIndexModule extends AbstractModule {
private static class DummyChangeIndexFactory implements ChangeIndex.Factory {
@Override
public ChangeIndex create(Schema<ChangeData> schema) {
throw new UnsupportedOperationException();
}
}
@Override
protected void configure() {
install(new IndexModule(1));
bind(IndexConfig.class).toInstance(IndexConfig.createDefault());
bind(Index.class).toInstance(new DummyChangeIndex());
bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory());
}
}

71
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexDefinition.java Normal file
View File

@@ -0,0 +1,71 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index;
import com.google.common.collect.ImmutableSortedMap;
/**
* Definition of an index over a Gerrit data type.
* <p>
* An <em>index</em> includes a set of schema definitions along with the
* specific implementations used to query the secondary index implementation in
* a running server. If you are just interested in the static definition of one
* or more schemas, see the implementations of {@link SchemaDefinitions}.
*/
public abstract class IndexDefinition<K, V, I extends Index<K, V>> {
public interface IndexFactory<K, V, I extends Index<K, V>> {
I create(Schema<V> schema);
}
private final SchemaDefinitions<V> schemaDefs;
private final IndexCollection<K, V, I> indexCollection;
private final IndexFactory<K, V, I> indexFactory;
private final SiteIndexer<K, V, I> siteIndexer;
protected IndexDefinition(
SchemaDefinitions<V> schemaDefs,
IndexCollection<K, V, I> indexCollection,
IndexFactory<K, V, I> indexFactory,
SiteIndexer<K, V, I> siteIndexer) {
this.schemaDefs = schemaDefs;
this.indexCollection = indexCollection;
this.indexFactory = indexFactory;
this.siteIndexer = siteIndexer;
}
public final String getName() {
return schemaDefs.getName();
}
public final ImmutableSortedMap<Integer, Schema<V>> getSchemas() {
return schemaDefs.getSchemas();
}
public final Schema<V> getLatest() {
return schemaDefs.getLatest();
}
public final IndexCollection<K, V, I> getIndexCollection() {
return indexCollection;
}
public final IndexFactory<K, V, I> getIndexFactory() {
return indexFactory;
}
public final SiteIndexer<K, V, I> getSiteIndexer() {
return siteIndexer;
}
}

41
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
View File

@@ -17,21 +17,31 @@ package com.google.gerrit.server.index;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeIndexDefintion;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.index.change.IndexRewriter;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import org.eclipse.jgit.lib.Config;
import java.util.Collection;
import java.util.Set;
/**
* Module for non-indexer-specific secondary index setup.
* <p>
@@ -43,6 +53,10 @@ public class IndexModule extends LifecycleModule {
LUCENE
}
public static final ImmutableCollection<SchemaDefinitions<?>> ALL_SCHEMA_DEFS =
ImmutableList.<SchemaDefinitions<?>> of(
ChangeSchemaDefinitions.INSTANCE);
/** Type of secondary index. */
public static IndexType getIndexType(Injector injector) {
Config cfg = injector.getInstance(
@@ -74,6 +88,33 @@ public class IndexModule extends LifecycleModule {
factory(ChangeIndexer.Factory.class);
}
@Provides
Collection<IndexDefinition<?, ?, ?>> getIndexDefinitions(
ChangeIndexDefintion changes) {
Collection<IndexDefinition<?, ?, ?>> result =
ImmutableList.<IndexDefinition<?, ?, ?>> of(changes);
Set<String> expected = FluentIterable.from(ALL_SCHEMA_DEFS)
.transform(new Function<SchemaDefinitions<?>, String>() {
@Override
public String apply(SchemaDefinitions<?> in) {
return in.getName();
}
}).toSet();
Set<String> actual = FluentIterable.from(result)
.transform(new Function<IndexDefinition<?, ?, ?>, String>() {
@Override
public String apply(IndexDefinition<?, ?, ?> in) {
return in.getName();
}
}).toSet();
if (!expected.equals(actual)) {
throw new ProvisionException(
"need index definitions for all schemas: "
+ expected + " != " + actual);
}
return result;
}
@Provides
@Singleton
ChangeIndexer getChangeIndexer(

47
gerrit-lucene/src/main/java/com/google/gerrit/lucene/OnlineReindexer.java → gerrit-server/src/main/java/com/google/gerrit/server/index/OnlineReindexer.java
View File

@@ -12,18 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.Lists;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.SiteIndexer;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,30 +25,21 @@ import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class OnlineReindexer {
public class OnlineReindexer<K, V, I extends Index<K, V>> {
private static final Logger log = LoggerFactory
.getLogger(OnlineReindexer.class);
public interface Factory {
OnlineReindexer create(int version);
}
private final ChangeIndexCollection indexes;
private final SiteIndexer batchIndexer;
private final ProjectCache projectCache;
private final IndexCollection<K, V, I> indexes;
private final SiteIndexer<K, V, I> batchIndexer;
private final int version;
private ChangeIndex index;
private I index;
private final AtomicBoolean running = new AtomicBoolean();
@Inject
OnlineReindexer(
ChangeIndexCollection indexes,
SiteIndexer batchIndexer,
ProjectCache projectCache,
@Assisted int version) {
this.indexes = indexes;
this.batchIndexer = batchIndexer;
this.projectCache = projectCache;
public OnlineReindexer(
IndexDefinition<K, V, I> def,
int version) {
this.indexes = def.getIndexCollection();
this.batchIndexer = def.getSiteIndexer();
this.version = version;
}
@@ -94,8 +78,7 @@ public class OnlineReindexer {
"not an active write schema version: %s", version);
log.info("Starting online reindex from schema version {} to {}",
version(indexes.getSearchIndex()), version(index));
SiteIndexer.Result result =
batchIndexer.indexAll(index, projectCache.all());
SiteIndexer.Result result = batchIndexer.indexAll(index);
if (!result.success()) {
log.error("Online reindex of schema version {} failed. Successfully"
+ " indexed {} changes, failed to index {} changes",
@@ -106,7 +89,7 @@ public class OnlineReindexer {
activateIndex();
}
void activateIndex() {
public void activateIndex() {
indexes.setSearchIndex(index);
log.info("Using schema version {}", version(index));
try {
@@ -115,13 +98,13 @@ public class OnlineReindexer {
log.warn("Error activating new schema version {}", version(index));
}
List<ChangeIndex> toRemove = Lists.newArrayListWithExpectedSize(1);
for (ChangeIndex i : indexes.getWriteIndexes()) {
List<I> toRemove = Lists.newArrayListWithExpectedSize(1);
for (I i : indexes.getWriteIndexes()) {
if (version(i) != version(index)) {
toRemove.add(i);
}
}
for (ChangeIndex i : toRemove) {
for (I i : toRemove) {
try {
i.markReady(false);
indexes.removeWriteIndex(version(i));

58
gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaDefinitions.java Normal file
View File

@@ -0,0 +1,58 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.ImmutableSortedMap;
/**
* Definitions of the various schema versions over a given Gerrit data type.
* <p>
* A <em>schema</em> is a description of the fields that are indexed over the
* given data type. This class contains all the versions of a schema defined
* over its data type, exposed as a map of version number to schema definition.
* If you are interested in the classes responsible for backend-specific runtime
* implementations, see the implementations of {@link IndexDefinition}.
*/
public abstract class SchemaDefinitions<V> {
private final String name;
private final ImmutableSortedMap<Integer, Schema<V>> schemas;
protected SchemaDefinitions(String name, Class<V> valueClass) {
this.name = checkNotNull(name);
this.schemas = SchemaUtil.schemasFromClass(getClass(), valueClass);
}
public final String getName() {
return name;
}
public final ImmutableSortedMap<Integer, Schema<V>> getSchemas() {
return schemas;
}
public final Schema<V> get(int version) {
Schema<V> schema = schemas.get(version);
checkArgument(schema != null,
"Unrecognized %s schema version: %s", name, version);
return schema;
}
public final Schema<V> getLatest() {
return schemas.lastEntry().getValue();
}
}

37
gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaUtil.java
View File

@@ -19,8 +19,8 @@ import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.base.CharMatcher;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -30,16 +30,16 @@ import org.eclipse.jgit.lib.PersonIdent;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class SchemaUtil {
public static <V> ImmutableMap<Integer, Schema<V>> schemasFromClass(
public static <V> ImmutableSortedMap<Integer, Schema<V>> schemasFromClass(
Class<?> schemasClass, Class<V> valueClass) {
Map<Integer, Schema<V>> schemas = Maps.newTreeMap();
Map<Integer, Schema<V>> schemas = Maps.newHashMap();
for (Field f : schemasClass.getDeclaredFields()) {
if (Modifier.isStatic(f.getModifiers())
&& Modifier.isFinal(f.getModifiers())
@@ -65,7 +65,7 @@ public class SchemaUtil {
if (schemas.isEmpty()) {
throw new ExceptionInInitializerError("no ChangeSchemas found");
}
return ImmutableMap.copyOf(schemas);
return ImmutableSortedMap.copyOf(schemas);
}
public static <V> Schema<V> schema(Collection<FieldDef<V, ?>> fields) {
@@ -81,13 +81,28 @@ public class SchemaUtil {
if (person == null) {
return ImmutableSet.of();
}
HashSet<String> parts = Sets.newHashSet();
String email = person.getEmailAddress().toLowerCase();
parts.add(email);
parts.addAll(Arrays.asList(email.split("@")));
return getPersonParts(
person.getName(),
Collections.singleton(person.getEmailAddress()));
}
public static Set<String> getPersonParts(String name,
Iterable<String> emails) {
Splitter at = Splitter.on('@');
Splitter s = Splitter.on(CharMatcher.anyOf("@.- ")).omitEmptyStrings();
Iterables.addAll(parts, s.split(email));
Iterables.addAll(parts, s.split(person.getName().toLowerCase()));
HashSet<String> parts = Sets.newHashSet();
for (String email : emails) {
if (email == null) {
continue;
}
String lowerEmail = email.toLowerCase();
parts.add(lowerEmail);
Iterables.addAll(parts, at.split(lowerEmail));
Iterables.addAll(parts, s.split(lowerEmail));
}
if (name != null) {
Iterables.addAll(parts, s.split(name.toLowerCase()));
}
return parts;
}

381
gerrit-server/src/main/java/com/google/gerrit/server/index/SiteIndexer.java
View File

@@ -1,4 +1,4 @@
// Copyright (C) 2013 The Android Open Source Project
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,81 +14,18 @@
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
import static org.eclipse.jgit.lib.RefDatabase.ALL;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.MergeUtil;
import com.google.gerrit.server.git.MultiProgressMonitor;
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.patch.PatchListLoader;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.merge.ThreeWayMergeStrategy;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.util.io.DisabledOutputStream;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
public class SiteIndexer {
private static final Logger log =
LoggerFactory.getLogger(SiteIndexer.class);
public static class Result {
public interface SiteIndexer<K, V, I extends Index<K, V>> {
public class Result {
private final long elapsedNanos;
private final boolean success;
private final int done;
private final int failed;
private Result(Stopwatch sw, boolean success, int done, int failed) {
public Result(Stopwatch sw, boolean success, int done, int failed) {
this.elapsedNanos = sw.elapsed(TimeUnit.NANOSECONDS);
this.success = success;
this.done = done;
@@ -112,313 +49,5 @@ public class SiteIndexer {
}
}
private final SchemaFactory<ReviewDb> schemaFactory;
private final ChangeData.Factory changeDataFactory;
private final GitRepositoryManager repoManager;
private final ListeningExecutorService executor;
private final ChangeIndexer.Factory indexerFactory;
private final ChangeNotes.Factory notesFactory;
private final ThreeWayMergeStrategy mergeStrategy;
private int numChanges = -1;
private OutputStream progressOut = NullOutputStream.INSTANCE;
private PrintWriter verboseWriter =
new PrintWriter(NullOutputStream.INSTANCE);
@Inject
SiteIndexer(SchemaFactory<ReviewDb> schemaFactory,
ChangeData.Factory changeDataFactory,
GitRepositoryManager repoManager,
@IndexExecutor(BATCH) ListeningExecutorService executor,
ChangeIndexer.Factory indexerFactory,
ChangeNotes.Factory notesFactory,
@GerritServerConfig Config config) {
this.schemaFactory = schemaFactory;
this.changeDataFactory = changeDataFactory;
this.repoManager = repoManager;
this.executor = executor;
this.indexerFactory = indexerFactory;
this.notesFactory = notesFactory;
this.mergeStrategy = MergeUtil.getMergeStrategy(config);
}
public SiteIndexer setNumChanges(int num) {
numChanges = num;
return this;
}
public SiteIndexer setProgressOut(OutputStream out) {
progressOut = checkNotNull(out);
return this;
}
public SiteIndexer setVerboseOut(OutputStream out) {
verboseWriter = new PrintWriter(checkNotNull(out));
return this;
}
public Result indexAll(ChangeIndex index,
Iterable<Project.NameKey> projects) {
Stopwatch sw = Stopwatch.createStarted();
final MultiProgressMonitor mpm =
new MultiProgressMonitor(progressOut, "Reindexing changes");
final Task projTask = mpm.beginSubTask("projects",
(projects instanceof Collection)
? ((Collection<?>) projects).size()
: MultiProgressMonitor.UNKNOWN);
final Task doneTask = mpm.beginSubTask(null,
numChanges >= 0 ? numChanges : MultiProgressMonitor.UNKNOWN);
final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN);
final List<ListenableFuture<?>> futures = Lists.newArrayList();
final AtomicBoolean ok = new AtomicBoolean(true);
for (final Project.NameKey project : projects) {
final ListenableFuture<?> future = executor.submit(reindexProject(
indexerFactory.create(executor, index), project, doneTask, failedTask,
verboseWriter));
futures.add(future);
future.addListener(new Runnable() {
@Override
public void run() {
try {
future.get();
} catch (ExecutionException | InterruptedException e) {
fail(project, e);
} catch (RuntimeException e) {
failAndThrow(project, e);
} catch (Error e) {
// Can't join with RuntimeException because "RuntimeException |
// Error" becomes Throwable, which messes with signatures.
failAndThrow(project, e);
} finally {
projTask.update(1);
}
}
private void fail(Project.NameKey project, Throwable t) {
log.error("Failed to index project " + project, t);
ok.set(false);
}
private void failAndThrow(Project.NameKey project, RuntimeException e) {
fail(project, e);
throw e;
}
private void failAndThrow(Project.NameKey project, Error e) {
fail(project, e);
throw e;
}
}, MoreExecutors.directExecutor());
}
try {
mpm.waitFor(Futures.transformAsync(Futures.successfulAsList(futures),
new AsyncFunction<List<?>, Void>() {
@Override
public ListenableFuture<Void> apply(List<?> input) {
mpm.end();
return Futures.immediateFuture(null);
}
}));
} catch (ExecutionException e) {
log.error("Error in batch indexer", e);
ok.set(false);
}
// If too many changes failed, maybe there was a bug in the indexer. Don't
// trust the results. This is not an exact percentage since we bump the same
// failure counter if a project can't be read, but close enough.
int nFailed = failedTask.getCount();
int nTotal = nFailed + doneTask.getCount();
double pctFailed = ((double) nFailed) / nTotal * 100;
if (pctFailed > 10) {
log.error("Failed {}/{} changes ({}%); not marking new index as ready",
nFailed, nTotal, Math.round(pctFailed));
ok.set(false);
}
return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount());
}
private Callable<Void> reindexProject(final ChangeIndexer indexer,
final Project.NameKey project, final Task done, final Task failed,
final PrintWriter verboseWriter) {
return new Callable<Void>() {
@Override
public Void call() throws Exception {
Multimap<ObjectId, ChangeData> byId = ArrayListMultimap.create();
// TODO(dborowitz): Opening all repositories in a live server may be
// wasteful; see if we can determine which ones it is safe to close
// with RepositoryCache.close(repo).
try (Repository repo = repoManager.openRepository(project);
ReviewDb db = schemaFactory.open()) {
Map<String, Ref> refs = repo.getRefDatabase().getRefs(ALL);
for (ChangeNotes cn : notesFactory.scan(repo, db, project)) {
Ref r = refs.get(cn.getChange().currentPatchSetId().toRefName());
if (r != null) {
byId.put(r.getObjectId(), changeDataFactory.create(db, cn));
}
}
new ProjectIndexer(indexer,
mergeStrategy,
byId,
repo,
done,
failed,
verboseWriter).call();
} catch (RepositoryNotFoundException rnfe) {
log.error(rnfe.getMessage());
}
return null;
}
@Override
public String toString() {
return "Index all changes of project " + project.get();
}
};
}
private static class ProjectIndexer implements Callable<Void> {
private final ChangeIndexer indexer;
private final ThreeWayMergeStrategy mergeStrategy;
private final Multimap<ObjectId, ChangeData> byId;
private final ProgressMonitor done;
private final ProgressMonitor failed;
private final PrintWriter verboseWriter;
private final Repository repo;
private RevWalk walk;
private ProjectIndexer(ChangeIndexer indexer,
ThreeWayMergeStrategy mergeStrategy,
Multimap<ObjectId, ChangeData> changesByCommitId,
Repository repo,
ProgressMonitor done,
ProgressMonitor failed,
PrintWriter verboseWriter) {
this.indexer = indexer;
this.mergeStrategy = mergeStrategy;
this.byId = changesByCommitId;
this.repo = repo;
this.done = done;
this.failed = failed;
this.verboseWriter = verboseWriter;
}
@Override
public Void call() throws Exception {
walk = new RevWalk(repo);
try {
// Walk only refs first to cover as many changes as we can without having
// to mark every single change.
for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) {
RevObject o = walk.parseAny(ref.getObjectId());
if (o instanceof RevCommit) {
walk.markStart((RevCommit) o);
}
}
RevCommit bCommit;
while ((bCommit = walk.next()) != null && !byId.isEmpty()) {
if (byId.containsKey(bCommit)) {
getPathsAndIndex(bCommit);
byId.removeAll(bCommit);
}
}
for (ObjectId id : byId.keySet()) {
getPathsAndIndex(id);
}
} finally {
walk.close();
}
return null;
}
private void getPathsAndIndex(ObjectId b) throws Exception {
List<ChangeData> cds = Lists.newArrayList(byId.get(b));
try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) {
RevCommit bCommit = walk.parseCommit(b);
RevTree bTree = bCommit.getTree();
RevTree aTree = aFor(bCommit, walk);
df.setRepository(repo);
if (!cds.isEmpty()) {
List<String> paths = (aTree != null)
? getPaths(df.scan(aTree, bTree))
: Collections.<String>emptyList();
Iterator<ChangeData> cdit = cds.iterator();
for (ChangeData cd ; cdit.hasNext(); cdit.remove()) {
cd = cdit.next();
try {
cd.setCurrentFilePaths(paths);
indexer.index(cd);
done.update(1);
if (verboseWriter != null) {
verboseWriter.println("Reindexed change " + cd.getId());
}
} catch (Exception e) {
fail("Failed to index change " + cd.getId(), true, e);
}
}
}
} catch (Exception e) {
fail("Failed to index commit " + b.name(), false, e);
for (ChangeData cd : cds) {
fail("Failed to index change " + cd.getId(), true, null);
}
}
}
private List<String> getPaths(List<DiffEntry> filenames) {
Set<String> paths = Sets.newTreeSet();
for (DiffEntry e : filenames) {
if (e.getOldPath() != null) {
paths.add(e.getOldPath());
}
if (e.getNewPath() != null) {
paths.add(e.getNewPath());
}
}
return ImmutableList.copyOf(paths);
}
private RevTree aFor(RevCommit b, RevWalk walk) throws IOException {
switch (b.getParentCount()) {
case 0:
return walk.parseTree(emptyTree());
case 1:
RevCommit a = b.getParent(0);
walk.parseBody(a);
return walk.parseTree(a.getTree());
case 2:
return PatchListLoader.automerge(repo, walk, b, mergeStrategy);
default:
return null;
}
}
private ObjectId emptyTree() throws IOException {
try (ObjectInserter oi = repo.newObjectInserter()) {
ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {});
oi.flush();
return id;
}
}
private void fail(String error, boolean failed, Exception e) {
if (failed) {
this.failed.update(1);
}
if (e != null) {
log.warn(error, e);
} else {
log.warn(error);
}
if (verboseWriter != null) {
verboseWriter.println(error);
}
}
}
Result indexAll(I index);
}

140
gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountField.java Normal file
View File

@@ -0,0 +1,140 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.account;
import com.google.common.base.Function;
import com.google.common.base.Predicates;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.gerrit.reviewdb.client.AccountExternalId;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.SchemaUtil;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.Set;
/** Secondary index schemas for accounts. */
public class AccountField {
public static final FieldDef<AccountState, Integer> ID =
new FieldDef.Single<AccountState, Integer>(
"id", FieldType.INTEGER, true) {
@Override
public Integer get(AccountState input, FillArgs args) {
return input.getAccount().getId().get();
}
};
public static final FieldDef<AccountState, Iterable<String>> EXTERNAL_ID =
new FieldDef.Repeatable<AccountState, String>(
"external_id", FieldType.EXACT, false) {
@Override
public Iterable<String> get(AccountState input, FillArgs args) {
return Iterables.transform(
input.getExternalIds(),
new Function<AccountExternalId, String>() {
@Override
public String apply(AccountExternalId in) {
return in.getKey().get();
}
});
}
};
/** Fuzzy prefix match on name and email parts. */
public static final FieldDef<AccountState, Iterable<String>> NAME_PART =
new FieldDef.Repeatable<AccountState, String>(
"name", FieldType.PREFIX, false) {
@Override
public Iterable<String> get(AccountState input, FillArgs args) {
String fullName = input.getAccount().getFullName();
Set<String> parts = SchemaUtil.getPersonParts(
fullName,
Iterables.transform(
input.getExternalIds(),
new Function<AccountExternalId, String>() {
@Override
public String apply(AccountExternalId in) {
return in.getEmailAddress();
}
}));
// Additional values not currently added by getPersonParts.
// TODO(dborowitz): Move to getPersonParts and remove this hack.
if (fullName != null) {
parts.add(fullName);
}
return parts;
}
};
public static final FieldDef<AccountState, String> ACTIVE =
new FieldDef.Single<AccountState, String>(
"inactive", FieldType.EXACT, false) {
@Override
public String get(AccountState input, FillArgs args) {
return input.getAccount().isActive() ? "1" : "0";
}
};
public static final FieldDef<AccountState, Iterable<String>> EMAIL =
new FieldDef.Repeatable<AccountState, String>(
"email", FieldType.PREFIX, false) {
@Override
public Iterable<String> get(AccountState input, FillArgs args) {
return FluentIterable.from(input.getExternalIds())
.transform(
new Function<AccountExternalId, String>() {
@Override
public String apply(AccountExternalId in) {
return in.getEmailAddress();
}
})
.append(
Collections.singleton(input.getAccount().getPreferredEmail()))
.filter(Predicates.notNull())
.transform(
new Function<String, String>() {
@Override
public String apply(String in) {
return in.toLowerCase();
}
});
}
};
public static final FieldDef<AccountState, Timestamp> REGISTERED =
new FieldDef.Single<AccountState, Timestamp>(
"registered", FieldType.TIMESTAMP, false) {
@Override
public Timestamp get(AccountState input, FillArgs args) {
return input.getAccount().getRegisteredOn();
}
};
public static final FieldDef<AccountState, String> USERNAME =
new FieldDef.Single<AccountState, String>(
"username", null, false) {
@Override
public String get(AccountState input, FillArgs args) {
return input.getUserName().toLowerCase();
}
};
private AccountField() {
}
}

62
gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountSchemas.java Normal file
View File

@@ -0,0 +1,62 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.account;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.SchemaUtil;
import java.util.Collection;
public class AccountSchemas {
static final Schema<AccountState> V1 = schema(
AccountField.ID,
AccountField.ACTIVE,
AccountField.EMAIL,
AccountField.EXTERNAL_ID,
AccountField.NAME_PART,
AccountField.REGISTERED,
AccountField.USERNAME);
private static Schema<AccountState> schema(
Collection<FieldDef<AccountState, ?>> fields) {
return new Schema<>(ImmutableList.copyOf(fields));
}
@SafeVarargs
private static Schema<AccountState> schema(
FieldDef<AccountState, ?>... fields) {
return schema(ImmutableList.copyOf(fields));
}
public static final ImmutableMap<Integer, Schema<AccountState>> ALL =
SchemaUtil.schemasFromClass(AccountSchemas.class, AccountState.class);
public static Schema<AccountState> get(int version) {
Schema<AccountState> schema = ALL.get(version);
checkArgument(schema != null, "Unrecognized schema version: %s", version);
return schema;
}
public static Schema<AccountState> getLatest() {
return Iterables.getLast(ALL.values());
}
}

404
gerrit-server/src/main/java/com/google/gerrit/server/index/change/AllChangesIndexer.java Normal file
View File

@@ -0,0 +1,404 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.change;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
import static org.eclipse.jgit.lib.RefDatabase.ALL;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.MergeUtil;
import com.google.gerrit.server.git.MultiProgressMonitor;
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.SiteIndexer;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.patch.PatchListLoader;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.merge.ThreeWayMergeStrategy;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.util.io.DisabledOutputStream;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
public class AllChangesIndexer
implements SiteIndexer<Change.Id, ChangeData, ChangeIndex> {
private static final Logger log =
LoggerFactory.getLogger(AllChangesIndexer.class);
private final SchemaFactory<ReviewDb> schemaFactory;
private final ChangeData.Factory changeDataFactory;
private final GitRepositoryManager repoManager;
private final ListeningExecutorService executor;
private final ChangeIndexer.Factory indexerFactory;
private final ChangeNotes.Factory notesFactory;
private final ProjectCache projectCache;
private final ThreeWayMergeStrategy mergeStrategy;
private int totalWork = -1;
private OutputStream progressOut = NullOutputStream.INSTANCE;
private PrintWriter verboseWriter =
new PrintWriter(NullOutputStream.INSTANCE);
@Inject
AllChangesIndexer(SchemaFactory<ReviewDb> schemaFactory,
ChangeData.Factory changeDataFactory,
GitRepositoryManager repoManager,
@IndexExecutor(BATCH) ListeningExecutorService executor,
ChangeIndexer.Factory indexerFactory,
ChangeNotes.Factory notesFactory,
@GerritServerConfig Config config,
ProjectCache projectCache) {
this.schemaFactory = schemaFactory;
this.changeDataFactory = changeDataFactory;
this.repoManager = repoManager;
this.executor = executor;
this.indexerFactory = indexerFactory;
this.notesFactory = notesFactory;
this.projectCache = projectCache;
this.mergeStrategy = MergeUtil.getMergeStrategy(config);
}
public AllChangesIndexer setTotalWork(int num) {
totalWork = num;
return this;
}
public AllChangesIndexer setProgressOut(OutputStream out) {
progressOut = checkNotNull(out);
return this;
}
public AllChangesIndexer setVerboseOut(OutputStream out) {
verboseWriter = new PrintWriter(checkNotNull(out));
return this;
}
@Override
public Result indexAll(ChangeIndex index) {
return indexAll(index, projectCache.all());
}
public SiteIndexer.Result indexAll(ChangeIndex index,
Iterable<Project.NameKey> projects) {
Stopwatch sw = Stopwatch.createStarted();
final MultiProgressMonitor mpm =
new MultiProgressMonitor(progressOut, "Reindexing changes");
final Task projTask = mpm.beginSubTask("projects",
(projects instanceof Collection)
? ((Collection<?>) projects).size()
: MultiProgressMonitor.UNKNOWN);
final Task doneTask = mpm.beginSubTask(null,
totalWork >= 0 ? totalWork : MultiProgressMonitor.UNKNOWN);
final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN);
final List<ListenableFuture<?>> futures = Lists.newArrayList();
final AtomicBoolean ok = new AtomicBoolean(true);
for (final Project.NameKey project : projects) {
final ListenableFuture<?> future = executor.submit(reindexProject(
indexerFactory.create(executor, index), project, doneTask, failedTask,
verboseWriter));
futures.add(future);
future.addListener(new Runnable() {
@Override
public void run() {
try {
future.get();
} catch (ExecutionException | InterruptedException e) {
fail(project, e);
} catch (RuntimeException e) {
failAndThrow(project, e);
} catch (Error e) {
// Can't join with RuntimeException because "RuntimeException |
// Error" becomes Throwable, which messes with signatures.
failAndThrow(project, e);
} finally {
projTask.update(1);
}
}
private void fail(Project.NameKey project, Throwable t) {
log.error("Failed to index project " + project, t);
ok.set(false);
}
private void failAndThrow(Project.NameKey project, RuntimeException e) {
fail(project, e);
throw e;
}
private void failAndThrow(Project.NameKey project, Error e) {
fail(project, e);
throw e;
}
}, MoreExecutors.directExecutor());
}
try {
mpm.waitFor(Futures.transformAsync(Futures.successfulAsList(futures),
new AsyncFunction<List<?>, Void>() {
@Override
public ListenableFuture<Void> apply(List<?> input) {
mpm.end();
return Futures.immediateFuture(null);
}
}));
} catch (ExecutionException e) {
log.error("Error in batch indexer", e);
ok.set(false);
}
// If too many changes failed, maybe there was a bug in the indexer. Don't
// trust the results. This is not an exact percentage since we bump the same
// failure counter if a project can't be read, but close enough.
int nFailed = failedTask.getCount();
int nTotal = nFailed + doneTask.getCount();
double pctFailed = ((double) nFailed) / nTotal * 100;
if (pctFailed > 10) {
log.error("Failed {}/{} changes ({}%); not marking new index as ready",
nFailed, nTotal, Math.round(pctFailed));
ok.set(false);
}
return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount());
}
private Callable<Void> reindexProject(final ChangeIndexer indexer,
final Project.NameKey project, final Task done, final Task failed,
final PrintWriter verboseWriter) {
return new Callable<Void>() {
@Override
public Void call() throws Exception {
Multimap<ObjectId, ChangeData> byId = ArrayListMultimap.create();
// TODO(dborowitz): Opening all repositories in a live server may be
// wasteful; see if we can determine which ones it is safe to close
// with RepositoryCache.close(repo).
try (Repository repo = repoManager.openRepository(project);
ReviewDb db = schemaFactory.open()) {
Map<String, Ref> refs = repo.getRefDatabase().getRefs(ALL);
for (ChangeNotes cn : notesFactory.scan(repo, db, project)) {
Ref r = refs.get(cn.getChange().currentPatchSetId().toRefName());
if (r != null) {
byId.put(r.getObjectId(), changeDataFactory.create(db, cn));
}
}
new ProjectIndexer(indexer,
mergeStrategy,
byId,
repo,
done,
failed,
verboseWriter).call();
} catch (RepositoryNotFoundException rnfe) {
log.error(rnfe.getMessage());
}
return null;
}
@Override
public String toString() {
return "Index all changes of project " + project.get();
}
};
}
private static class ProjectIndexer implements Callable<Void> {
private final ChangeIndexer indexer;
private final ThreeWayMergeStrategy mergeStrategy;
private final Multimap<ObjectId, ChangeData> byId;
private final ProgressMonitor done;
private final ProgressMonitor failed;
private final PrintWriter verboseWriter;
private final Repository repo;
private RevWalk walk;
private ProjectIndexer(ChangeIndexer indexer,
ThreeWayMergeStrategy mergeStrategy,
Multimap<ObjectId, ChangeData> changesByCommitId,
Repository repo,
ProgressMonitor done,
ProgressMonitor failed,
PrintWriter verboseWriter) {
this.indexer = indexer;
this.mergeStrategy = mergeStrategy;
this.byId = changesByCommitId;
this.repo = repo;
this.done = done;
this.failed = failed;
this.verboseWriter = verboseWriter;
}
@Override
public Void call() throws Exception {
walk = new RevWalk(repo);
try {
// Walk only refs first to cover as many changes as we can without having
// to mark every single change.
for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) {
RevObject o = walk.parseAny(ref.getObjectId());
if (o instanceof RevCommit) {
walk.markStart((RevCommit) o);
}
}
RevCommit bCommit;
while ((bCommit = walk.next()) != null && !byId.isEmpty()) {
if (byId.containsKey(bCommit)) {
getPathsAndIndex(bCommit);
byId.removeAll(bCommit);
}
}
for (ObjectId id : byId.keySet()) {
getPathsAndIndex(id);
}
} finally {
walk.close();
}
return null;
}
private void getPathsAndIndex(ObjectId b) throws Exception {
List<ChangeData> cds = Lists.newArrayList(byId.get(b));
try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) {
RevCommit bCommit = walk.parseCommit(b);
RevTree bTree = bCommit.getTree();
RevTree aTree = aFor(bCommit, walk);
df.setRepository(repo);
if (!cds.isEmpty()) {
List<String> paths = (aTree != null)
? getPaths(df.scan(aTree, bTree))
: Collections.<String>emptyList();
Iterator<ChangeData> cdit = cds.iterator();
for (ChangeData cd ; cdit.hasNext(); cdit.remove()) {
cd = cdit.next();
try {
cd.setCurrentFilePaths(paths);
indexer.index(cd);
done.update(1);
if (verboseWriter != null) {
verboseWriter.println("Reindexed change " + cd.getId());
}
} catch (Exception e) {
fail("Failed to index change " + cd.getId(), true, e);
}
}
}
} catch (Exception e) {
fail("Failed to index commit " + b.name(), false, e);
for (ChangeData cd : cds) {
fail("Failed to index change " + cd.getId(), true, null);
}
}
}
private List<String> getPaths(List<DiffEntry> filenames) {
Set<String> paths = Sets.newTreeSet();
for (DiffEntry e : filenames) {
if (e.getOldPath() != null) {
paths.add(e.getOldPath());
}
if (e.getNewPath() != null) {
paths.add(e.getNewPath());
}
}
return ImmutableList.copyOf(paths);
}
private RevTree aFor(RevCommit b, RevWalk walk) throws IOException {
switch (b.getParentCount()) {
case 0:
return walk.parseTree(emptyTree());
case 1:
RevCommit a = b.getParent(0);
walk.parseBody(a);
return walk.parseTree(a.getTree());
case 2:
return PatchListLoader.automerge(repo, walk, b, mergeStrategy);
default:
return null;
}
}
private ObjectId emptyTree() throws IOException {
try (ObjectInserter oi = repo.newObjectInserter()) {
ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {});
oi.flush();
return id;
}
}
private void fail(String error, boolean failed, Exception e) {
if (failed) {
this.failed.update(1);
}
if (e != null) {
log.warn(error, e);
} else {
log.warn(error);
}
if (verboseWriter != null) {
verboseWriter.println(error);
}
}
}
}

4
gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndex.java
View File

@@ -16,7 +16,11 @@ package com.google.gerrit.server.index.change;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.IndexDefinition;
import com.google.gerrit.server.query.change.ChangeData;
public interface ChangeIndex extends Index<Change.Id, ChangeData> {
public interface Factory extends
IndexDefinition.IndexFactory<Change.Id, ChangeData, ChangeIndex> {
}
}

33
gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndexDefintion.java Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.change;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.index.IndexDefinition;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
public class ChangeIndexDefintion
extends IndexDefinition<Change.Id, ChangeData, ChangeIndex> {
@Inject
ChangeIndexDefintion(
ChangeIndexCollection indexCollection,
ChangeIndex.Factory indexFactory,
AllChangesIndexer allChangesIndexer) {
super(ChangeSchemaDefinitions.INSTANCE, indexCollection, indexFactory,
allChangesIndexer);
}
}

24
gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemas.java → gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java
View File

@@ -1,4 +1,4 @@
// Copyright (C) 2013 The Android Open Source Project
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,17 +14,13 @@
package com.google.gerrit.server.index.change;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.gerrit.server.index.SchemaUtil.schema;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.SchemaUtil;
import com.google.gerrit.server.index.SchemaDefinitions;
import com.google.gerrit.server.query.change.ChangeData;
/** Secondary index schemas for changes. */
public class ChangeSchemas {
public class ChangeSchemaDefinitions extends SchemaDefinitions<ChangeData> {
@Deprecated
static final Schema<ChangeData> V25 = schema(
ChangeField.LEGACY_ID,
@@ -102,16 +98,10 @@ public class ChangeSchemas {
static final Schema<ChangeData> V27 = schema(V26.getFields().values());
public static final ImmutableMap<Integer, Schema<ChangeData>> ALL =
SchemaUtil.schemasFromClass(ChangeSchemas.class, ChangeData.class);
public static final ChangeSchemaDefinitions INSTANCE =
new ChangeSchemaDefinitions();
public static Schema<ChangeData> get(int version) {
Schema<ChangeData> schema = ALL.get(version);
checkArgument(schema != null, "Unrecognized schema version: %s", version);
return schema;
}
public static Schema<ChangeData> getLatest() {
return Iterables.getLast(ALL.values());
private ChangeSchemaDefinitions() {
super("changes", ChangeData.class);
}
}

25
gerrit-server/src/test/java/com/google/gerrit/testutil/InMemoryModule.java
View File

@@ -14,7 +14,6 @@
package com.google.gerrit.testutil;
import static com.google.common.base.Preconditions.checkState;
import static com.google.inject.Scopes.SINGLETON;
import com.google.common.util.concurrent.MoreExecutors;
@@ -49,7 +48,7 @@ import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
import com.google.gerrit.server.git.SendEmailExecutor;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gerrit.server.patch.DiffExecutor;
@@ -74,10 +73,12 @@ import com.google.inject.servlet.RequestScoped;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.PersonIdent;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
public class InMemoryModule extends FactoryModule {
@@ -97,8 +98,6 @@ public class InMemoryModule extends FactoryModule {
cfg.unset("cache", null, "directory");
cfg.setString("index", null, "type", "lucene");
cfg.setBoolean("index", "lucene", "testInmemory", true);
cfg.setInt("index", "lucene", "testVersion",
ChangeSchemas.getLatest().getVersion());
cfg.setInt("sendemail", null, "threadPoolSize", 0);
cfg.setBoolean("receive", null, "enableSignedPush", false);
cfg.setString("receive", null, "certNonceSeed", "sekret");
@@ -234,16 +233,20 @@ public class InMemoryModule extends FactoryModule {
private Module luceneIndexModule() {
try {
Map<String, Integer> singleVersions = new HashMap<>();
int version = cfg.getInt("index", "lucene", "testVersion", -1);
checkState(ChangeSchemas.ALL.containsKey(version),
"invalid index.lucene.testVersion %s", version);
if (version > 0) {
singleVersions.put(ChangeSchemaDefinitions.INSTANCE.getName(), version);
}
Class<?> clazz =
Class.forName("com.google.gerrit.lucene.LuceneIndexModule");
Constructor<?> c = clazz.getConstructor(Integer.class, int.class);
return (Module) c.newInstance(version, 0);
Method m = clazz.getMethod(
"singleVersionWithExplicitVersions", Map.class, int.class);
return (Module) m.invoke(null, singleVersions, 0);
} catch (ClassNotFoundException | SecurityException | NoSuchMethodException
| IllegalArgumentException | InstantiationException
| IllegalAccessException | InvocationTargetException e) {
| IllegalArgumentException | IllegalAccessException
| InvocationTargetException e) {
e.printStackTrace();
ProvisionException pe = new ProvisionException(e.getMessage());
pe.initCause(e);
throw pe;

2
gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java
View File

@@ -337,7 +337,7 @@ public class WebAppInitializer extends GuiceServletContextListener
private Module createIndexModule() {
switch (indexType) {
case LUCENE:
return new LuceneIndexModule();
return LuceneIndexModule.latestVersionWithOnlineUpgrade();
default:
throw new IllegalStateException("unsupported index.type = " + indexType);
}