diff --git a/gerrit-acceptance-framework/src/test/java/com/google/gerrit/acceptance/GerritServer.java b/gerrit-acceptance-framework/src/test/java/com/google/gerrit/acceptance/GerritServer.java index 6622286b7a..238567caae 100644 --- a/gerrit-acceptance-framework/src/test/java/com/google/gerrit/acceptance/GerritServer.java +++ b/gerrit-acceptance-framework/src/test/java/com/google/gerrit/acceptance/GerritServer.java @@ -26,7 +26,6 @@ import com.google.gerrit.pgm.Daemon; import com.google.gerrit.pgm.Init; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.git.AsyncReceiveCommits; -import com.google.gerrit.server.index.change.ChangeSchemas; import com.google.gerrit.server.ssh.NoSshModule; import com.google.gerrit.server.util.SocketUtil; import com.google.gerrit.server.util.SystemLog; @@ -142,8 +141,7 @@ public class GerritServer { cfg.setBoolean("index", "lucene", "testInmemory", true); cfg.setString("gitweb", null, "cgi", ""); daemon.setEnableHttpd(desc.httpd()); - daemon.setLuceneModule(new LuceneIndexModule( - ChangeSchemas.getLatest().getVersion(), 0)); + daemon.setLuceneModule(LuceneIndexModule.singleVersionAllLatest(0)); daemon.setDatabaseForTesting(ImmutableList.of( new InMemoryTestingDatabaseModule(cfg))); daemon.start(); diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java similarity index 67% rename from gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java rename to gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java index ac43363c90..5acdc25358 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java @@ -20,9 +20,22 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.gerrit.lucene.LuceneChangeIndex.GerritIndexWriterConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.FieldDef; +import com.google.gerrit.server.index.FieldDef.FillArgs; +import com.google.gerrit.server.index.FieldType; +import com.google.gerrit.server.index.Index; +import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.index.Schema.Values; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.TrackingIndexWriter; @@ -33,12 +46,13 @@ import org.apache.lucene.search.ReferenceManager.RefreshListener; import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.file.Path; +import java.sql.Timestamp; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -46,25 +60,45 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -/** Piece of the change index that is implemented as a separate Lucene index. */ -public class SubIndex { - private static final Logger log = LoggerFactory.getLogger(SubIndex.class); +/** Basic Lucene index implementation. */ +public abstract class AbstractLuceneIndex implements Index { + private static final Logger log = + LoggerFactory.getLogger(AbstractLuceneIndex.class); + static String sortFieldName(FieldDef f) { + return f.getName() + "_SORT"; + } + + public static void setReady(SitePaths sitePaths, int version, boolean ready) + throws IOException { + try { + // TODO(dborowitz): Totally broken for non-change indexes. + FileBasedConfig cfg = + LuceneVersionManager.loadGerritIndexConfig(sitePaths); + LuceneVersionManager.setReady(cfg, version, ready); + cfg.save(); + } catch (ConfigInvalidException e) { + throw new IOException(e); + } + } + + private final Schema schema; + private final SitePaths sitePaths; private final Directory dir; private final TrackingIndexWriter writer; private final ReferenceManager searcherManager; private final ControlledRealTimeReopenThread reopenThread; private final Set notDoneNrtFutures; - SubIndex(Path path, GerritIndexWriterConfig writerConfig, - SearcherFactory searcherFactory) throws IOException { - this(FSDirectory.open(path), path.getFileName().toString(), writerConfig, - searcherFactory); - } - - SubIndex(Directory dir, final String dirName, + AbstractLuceneIndex( + Schema schema, + SitePaths sitePaths, + Directory dir, + final String name, GerritIndexWriterConfig writerConfig, SearcherFactory searcherFactory) throws IOException { + this.schema = schema; + this.sitePaths = sitePaths; this.dir = dir; IndexWriter delegateWriter; long commitPeriod = writerConfig.getCommitWithinMs(); @@ -80,7 +114,7 @@ public class SubIndex { delegateWriter = autoCommitWriter; new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder() - .setNameFormat("Commit-%d " + dirName) + .setNameFormat("Commit-%d " + name) .setDaemon(true) .build()) .scheduleAtFixedRate(new Runnable() { @@ -92,14 +126,14 @@ public class SubIndex { autoCommitWriter.commit(); } } catch (IOException e) { - log.error("Error committing Lucene index " + dirName, e); + log.error("Error committing " + name + " Lucene index", e); } catch (OutOfMemoryError e) { - log.error("Error committing Lucene index " + dirName, e); + log.error("Error committing " + name + " Lucene index", e); try { autoCommitWriter.close(); } catch (IOException e2) { - log.error("SEVERE: Error closing Lucene index " + dirName - + " after OOM; index may be corrupted.", e); + log.error("SEVERE: Error closing " + name + + " Lucene index after OOM; index may be corrupted.", e); } } } @@ -115,7 +149,7 @@ public class SubIndex { writer, searcherManager, 0.500 /* maximum stale age (seconds) */, 0.010 /* minimum stale age (seconds) */); - reopenThread.setName("NRT " + dirName); + reopenThread.setName("NRT " + name); reopenThread.setPriority(Math.min( Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY)); @@ -145,7 +179,13 @@ public class SubIndex { reopenThread.start(); } - void close() { + @Override + public void markReady(boolean ready) throws IOException { + setReady(sitePaths, schema.getVersion(), ready); + } + + @Override + public void close() { reopenThread.close(); // Closing the reopen thread sets its generation to Long.MAX_VALUE, but we @@ -187,7 +227,8 @@ public class SubIndex { return new NrtFuture(writer.deleteDocuments(term)); } - void deleteAll() throws IOException { + @Override + public void deleteAll() throws IOException { writer.deleteAll(); } @@ -203,6 +244,55 @@ public class SubIndex { searcherManager.release(searcher); } + Document toDocument(V obj, FillArgs fillArgs) { + Document result = new Document(); + for (Values vs : schema.buildFields(obj, fillArgs)) { + if (vs.getValues() != null) { + add(result, vs); + } + } + return result; + } + + void add(Document doc, Values values) { + String name = values.getField().getName(); + FieldType type = values.getField().getType(); + Store store = store(values.getField()); + + if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) { + for (Object value : values.getValues()) { + doc.add(new IntField(name, (Integer) value, store)); + } + } else if (type == FieldType.LONG) { + for (Object value : values.getValues()) { + doc.add(new LongField(name, (Long) value, store)); + } + } else if (type == FieldType.TIMESTAMP) { + for (Object value : values.getValues()) { + doc.add(new LongField(name, ((Timestamp) value).getTime(), store)); + } + } else if (type == FieldType.EXACT + || type == FieldType.PREFIX) { + for (Object value : values.getValues()) { + doc.add(new StringField(name, (String) value, store)); + } + } else if (type == FieldType.FULL_TEXT) { + for (Object value : values.getValues()) { + doc.add(new TextField(name, (String) value, store)); + } + } else if (type == FieldType.STORED_ONLY) { + for (Object value : values.getValues()) { + doc.add(new StoredField(name, (byte[]) value)); + } + } else { + throw FieldType.badFieldType(type); + } + } + + private static Field.Store store(FieldDef f) { + return f.isStored() ? Field.Store.YES : Field.Store.NO; + } + private final class NrtFuture extends AbstractFuture { private final long gen; @@ -287,4 +377,9 @@ public class SubIndex { } } } + + @Override + public Schema getSchema() { + return schema; + } } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/ChangeSubIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/ChangeSubIndex.java new file mode 100644 index 0000000000..ad5349383a --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/ChangeSubIndex.java @@ -0,0 +1,98 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.lucene; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.gerrit.lucene.LuceneChangeIndex.ID_SORT_FIELD; +import static com.google.gerrit.lucene.LuceneChangeIndex.UPDATED_SORT_FIELD; + +import com.google.gerrit.reviewdb.client.Change; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.FieldDef; +import com.google.gerrit.server.index.QueryOptions; +import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.index.Schema.Values; +import com.google.gerrit.server.index.change.ChangeField; +import com.google.gerrit.server.index.change.ChangeIndex; +import com.google.gerrit.server.query.DataSource; +import com.google.gerrit.server.query.Predicate; +import com.google.gerrit.server.query.QueryParseException; +import com.google.gerrit.server.query.change.ChangeData; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.Timestamp; + +public class ChangeSubIndex extends AbstractLuceneIndex + implements ChangeIndex { + ChangeSubIndex( + Schema schema, + SitePaths sitePaths, + Path path, + GerritIndexWriterConfig writerConfig, + SearcherFactory searcherFactory) throws IOException { + this(schema, sitePaths, FSDirectory.open(path), + path.getFileName().toString(), writerConfig, searcherFactory); + } + + ChangeSubIndex( + Schema schema, + SitePaths sitePaths, + Directory dir, + String name, + GerritIndexWriterConfig writerConfig, + SearcherFactory searcherFactory) throws IOException { + super(schema, sitePaths, dir, name, writerConfig, searcherFactory); + } + + @Override + public void replace(ChangeData obj) throws IOException { + throw new UnsupportedOperationException( + "don't use ChangeSubIndex directly"); + } + + @Override + public void delete(Change.Id key) throws IOException { + throw new UnsupportedOperationException( + "don't use ChangeSubIndex directly"); + } + + @Override + public DataSource getSource(Predicate p, + QueryOptions opts) throws QueryParseException { + throw new UnsupportedOperationException( + "don't use ChangeSubIndex directly"); + } + + @Override + void add(Document doc, Values values) { + // Add separate DocValues fields for those fields needed for sorting. + FieldDef f = values.getField(); + if (f == ChangeField.LEGACY_ID) { + int v = (Integer) getOnlyElement(values.getValues()); + doc.add(new NumericDocValuesField(ID_SORT_FIELD, v)); + } else if (f == ChangeField.UPDATED) { + long t = ((Timestamp) getOnlyElement(values.getValues())).getTime(); + doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t)); + } + super.add(doc, values); + } +} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java new file mode 100644 index 0000000000..36145c6595 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java @@ -0,0 +1,77 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.lucene; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; + +import com.google.common.collect.ImmutableMap; +import com.google.gerrit.server.config.ConfigUtil; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.util.CharArraySet; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.eclipse.jgit.lib.Config; + +import java.util.Map; + +/** + * Combination of Lucene {@link IndexWriterConfig} with additional + * Gerrit-specific options. + */ +class GerritIndexWriterConfig { + private static final Map CUSTOM_CHAR_MAPPING = + ImmutableMap.of("_", " ", ".", " "); + + private final IndexWriterConfig luceneConfig; + private long commitWithinMs; + private final CustomMappingAnalyzer analyzer; + + GerritIndexWriterConfig(Config cfg, String name) { + analyzer = + new CustomMappingAnalyzer(new StandardAnalyzer( + CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING); + luceneConfig = new IndexWriterConfig(analyzer) + .setOpenMode(OpenMode.CREATE_OR_APPEND) + .setCommitOnClose(true); + double m = 1 << 20; + luceneConfig.setRAMBufferSizeMB(cfg.getLong( + "index", name, "ramBufferSize", + (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m); + luceneConfig.setMaxBufferedDocs(cfg.getInt( + "index", name, "maxBufferedDocs", + IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS)); + try { + commitWithinMs = + ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin", + MILLISECONDS.convert(5, MINUTES), MILLISECONDS); + } catch (IllegalArgumentException e) { + commitWithinMs = cfg.getLong("index", name, "commitWithin", 0); + } + } + + CustomMappingAnalyzer getAnalyzer() { + return analyzer; + } + + IndexWriterConfig getLuceneConfig() { + return luceneConfig; + } + + long getCommitWithinMs() { + return commitWithinMs; + } +} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index 5178b8fc14..8b508e21ac 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -15,17 +15,15 @@ package com.google.gerrit.lucene; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.gerrit.lucene.AbstractLuceneIndex.sortFieldName; +import static com.google.gerrit.lucene.LuceneVersionManager.CHANGES_PREFIX; import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE; import static com.google.gerrit.server.index.change.ChangeField.CHANGE; import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID; import static com.google.gerrit.server.index.change.ChangeField.PROJECT; import static com.google.gerrit.server.index.change.IndexRewriter.CLOSED_STATUSES; import static com.google.gerrit.server.index.change.IndexRewriter.OPEN_STATUSES; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -37,16 +35,12 @@ import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.reviewdb.client.PatchSet; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.server.ReviewDb; -import com.google.gerrit.server.config.ConfigUtil; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; -import com.google.gerrit.server.index.FieldDef; import com.google.gerrit.server.index.FieldDef.FillArgs; -import com.google.gerrit.server.index.FieldType; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.QueryOptions; import com.google.gerrit.server.index.Schema; -import com.google.gerrit.server.index.Schema.Values; import com.google.gerrit.server.index.change.ChangeField; import com.google.gerrit.server.index.change.ChangeField.ChangeProtoField; import com.google.gerrit.server.index.change.ChangeField.PatchSetApprovalProtoField; @@ -64,23 +58,10 @@ import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.IntField; -import org.apache.lucene.document.LongField; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -92,20 +73,16 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; -import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.storage.file.FileBasedConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; -import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -124,6 +101,11 @@ public class LuceneChangeIndex implements ChangeIndex { public static final String CHANGES_OPEN = "open"; public static final String CHANGES_CLOSED = "closed"; + static final String UPDATED_SORT_FIELD = + sortFieldName(ChangeField.UPDATED); + static final String ID_SORT_FIELD = + sortFieldName(ChangeField.LEGACY_ID); + private static final String ADDED_FIELD = ChangeField.ADDED.getName(); private static final String APPROVAL_FIELD = ChangeField.APPROVAL.getName(); private static final String CHANGE_FIELD = ChangeField.CHANGE.getName(); @@ -132,68 +114,13 @@ public class LuceneChangeIndex implements ChangeIndex { private static final String PATCH_SET_FIELD = ChangeField.PATCH_SET.getName(); private static final String REVIEWEDBY_FIELD = ChangeField.REVIEWEDBY.getName(); - private static final String UPDATED_SORT_FIELD = - sortFieldName(ChangeField.UPDATED); - private static final String ID_SORT_FIELD = - sortFieldName(ChangeField.LEGACY_ID); - private static final Map CUSTOM_CHAR_MAPPING = ImmutableMap.of( - "_", " ", ".", " "); - - public static void setReady(SitePaths sitePaths, int version, boolean ready) - throws IOException { - try { - FileBasedConfig cfg = - LuceneVersionManager.loadGerritIndexConfig(sitePaths); - LuceneVersionManager.setReady(cfg, version, ready); - cfg.save(); - } catch (ConfigInvalidException e) { - throw new IOException(e); - } + static Term idTerm(ChangeData cd) { + return QueryBuilder.intTerm(LEGACY_ID.getName(), cd.getId().get()); } - private static String sortFieldName(FieldDef f) { - return f.getName() + "_SORT"; - } - - static interface Factory { - LuceneChangeIndex create(Schema schema); - } - - static class GerritIndexWriterConfig { - private final IndexWriterConfig luceneConfig; - private long commitWithinMs; - - private GerritIndexWriterConfig(Config cfg, String name) { - CustomMappingAnalyzer analyzer = - new CustomMappingAnalyzer(new StandardAnalyzer( - CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING); - luceneConfig = new IndexWriterConfig(analyzer) - .setOpenMode(OpenMode.CREATE_OR_APPEND) - .setCommitOnClose(true); - double m = 1 << 20; - luceneConfig.setRAMBufferSizeMB(cfg.getLong( - "index", name, "ramBufferSize", - (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m); - luceneConfig.setMaxBufferedDocs(cfg.getInt( - "index", name, "maxBufferedDocs", - IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS)); - try { - commitWithinMs = - ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin", - MILLISECONDS.convert(5, MINUTES), MILLISECONDS); - } catch (IllegalArgumentException e) { - commitWithinMs = cfg.getLong("index", name, "commitWithin", 0); - } - } - - IndexWriterConfig getLuceneConfig() { - return luceneConfig; - } - - long getCommitWithinMs() { - return commitWithinMs; - } + static Term idTerm(Change.Id id) { + return QueryBuilder.intTerm(LEGACY_ID.getName(), id.get()); } private final SitePaths sitePaths; @@ -202,9 +129,9 @@ public class LuceneChangeIndex implements ChangeIndex { private final Provider db; private final ChangeData.Factory changeDataFactory; private final Schema schema; - private final QueryBuilder queryBuilder; - private final SubIndex openIndex; - private final SubIndex closedIndex; + private final QueryBuilder queryBuilder; + private final ChangeSubIndex openIndex; + private final ChangeSubIndex closedIndex; @AssistedInject LuceneChangeIndex( @@ -222,31 +149,25 @@ public class LuceneChangeIndex implements ChangeIndex { this.changeDataFactory = changeDataFactory; this.schema = schema; - CustomMappingAnalyzer analyzer = - new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET), - CUSTOM_CHAR_MAPPING); - queryBuilder = new QueryBuilder(analyzer); - - BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms", - BooleanQuery.getMaxClauseCount())); - GerritIndexWriterConfig openConfig = new GerritIndexWriterConfig(cfg, "changes_open"); GerritIndexWriterConfig closedConfig = new GerritIndexWriterConfig(cfg, "changes_closed"); + queryBuilder = new QueryBuilder<>(schema, openConfig.getAnalyzer()); + SearcherFactory searcherFactory = new SearcherFactory(); - if (cfg.getBoolean("index", "lucene", "testInmemory", false)) { - openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig, - searcherFactory); - closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig, - searcherFactory); + if (LuceneIndexModule.isInMemoryTest(cfg)) { + openIndex = new ChangeSubIndex(schema, sitePaths, new RAMDirectory(), + "ramOpen", openConfig, searcherFactory); + closedIndex = new ChangeSubIndex(schema, sitePaths, new RAMDirectory(), + "ramClosed", closedConfig, searcherFactory); } else { - Path dir = LuceneVersionManager.getDir(sitePaths, schema); - openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig, - searcherFactory); - closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig, - searcherFactory); + Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES_PREFIX, schema); + openIndex = new ChangeSubIndex(schema, sitePaths, + dir.resolve(CHANGES_OPEN), openConfig, searcherFactory); + closedIndex = new ChangeSubIndex(schema, sitePaths, + dir.resolve(CHANGES_CLOSED), closedConfig, searcherFactory); } } @@ -275,8 +196,10 @@ public class LuceneChangeIndex implements ChangeIndex { @Override public void replace(ChangeData cd) throws IOException { - Term id = QueryBuilder.idTerm(cd); - Document doc = toDocument(cd); + Term id = LuceneChangeIndex.idTerm(cd); + // toDocument is essentially static and doesn't depend on the specific + // sub-index, so just pick one. + Document doc = openIndex.toDocument(cd, fillArgs); try { if (cd.change().getStatus().isOpen()) { Futures.allAsList( @@ -294,7 +217,7 @@ public class LuceneChangeIndex implements ChangeIndex { @Override public void delete(Change.Id id) throws IOException { - Term idTerm = QueryBuilder.idTerm(id); + Term idTerm = LuceneChangeIndex.idTerm(id); try { Futures.allAsList( openIndex.delete(idTerm), @@ -314,7 +237,7 @@ public class LuceneChangeIndex implements ChangeIndex { public ChangeDataSource getSource(Predicate p, QueryOptions opts) throws QueryParseException { Set statuses = IndexRewriter.getPossibleStatus(p); - List indexes = Lists.newArrayListWithCapacity(2); + List indexes = Lists.newArrayListWithCapacity(2); if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) { indexes.add(openIndex); } @@ -326,7 +249,9 @@ public class LuceneChangeIndex implements ChangeIndex { @Override public void markReady(boolean ready) throws IOException { - setReady(sitePaths, schema.getVersion(), ready); + // Do not delegate to ChangeSubIndex#markReady, since changes have an + // additional level of directory nesting. + AbstractLuceneIndex.setReady(sitePaths, schema.getVersion(), ready); } private Sort getSort() { @@ -335,21 +260,17 @@ public class LuceneChangeIndex implements ChangeIndex { new SortField(ID_SORT_FIELD, SortField.Type.LONG, true)); } - public SubIndex getOpenChangesIndex() { - return openIndex; - } - - public SubIndex getClosedChangesIndex() { + public ChangeSubIndex getClosedChangesIndex() { return closedIndex; } private class QuerySource implements ChangeDataSource { - private final List indexes; + private final List indexes; private final Query query; private final QueryOptions opts; private final Sort sort; - private QuerySource(List indexes, Query query, QueryOptions opts, + private QuerySource(List indexes, Query query, QueryOptions opts, Sort sort) { this.indexes = indexes; this.query = checkNotNull(query, "null query from Lucene"); @@ -557,64 +478,4 @@ public class LuceneChangeIndex implements ChangeIndex { } return result; } - - private Document toDocument(ChangeData cd) { - Document result = new Document(); - for (Values vs : schema.buildFields(cd, fillArgs)) { - if (vs.getValues() != null) { - add(result, vs); - } - } - return result; - } - - private void add(Document doc, Values values) { - String name = values.getField().getName(); - FieldType type = values.getField().getType(); - Store store = store(values.getField()); - - FieldDef f = values.getField(); - - // Add separate DocValues fields for those fields needed for sorting. - if (f == ChangeField.LEGACY_ID) { - int v = (Integer) getOnlyElement(values.getValues()); - doc.add(new NumericDocValuesField(sortFieldName(f), v)); - } else if (f == ChangeField.UPDATED) { - long t = ((Timestamp) getOnlyElement(values.getValues())).getTime(); - doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t)); - } - - if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) { - for (Object value : values.getValues()) { - doc.add(new IntField(name, (Integer) value, store)); - } - } else if (type == FieldType.LONG) { - for (Object value : values.getValues()) { - doc.add(new LongField(name, (Long) value, store)); - } - } else if (type == FieldType.TIMESTAMP) { - for (Object value : values.getValues()) { - doc.add(new LongField(name, ((Timestamp) value).getTime(), store)); - } - } else if (type == FieldType.EXACT - || type == FieldType.PREFIX) { - for (Object value : values.getValues()) { - doc.add(new StringField(name, (String) value, store)); - } - } else if (type == FieldType.FULL_TEXT) { - for (Object value : values.getValues()) { - doc.add(new TextField(name, (String) value, store)); - } - } else if (type == FieldType.STORED_ONLY) { - for (Object value : values.getValues()) { - doc.add(new StoredField(name, (byte[]) value)); - } - } else { - throw FieldType.badFieldType(type); - } - } - - private static Field.Store store(FieldDef f) { - return f.isStored() ? Field.Store.YES : Field.Store.NO; - } } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java index ea11535fe9..98fc4c7023 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -14,40 +14,69 @@ package com.google.gerrit.lucene; +import com.google.common.collect.ImmutableMap; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.lifecycle.LifecycleModule; import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.index.Index; import com.google.gerrit.server.index.IndexConfig; +import com.google.gerrit.server.index.IndexDefinition; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.Schema; -import com.google.gerrit.server.index.change.ChangeIndexCollection; -import com.google.gerrit.server.index.change.ChangeSchemas; -import com.google.gerrit.server.query.change.ChangeData; +import com.google.gerrit.server.index.change.ChangeIndex; import com.google.inject.Inject; import com.google.inject.Provides; +import com.google.inject.ProvisionException; import com.google.inject.Singleton; +import com.google.inject.TypeLiteral; +import com.google.inject.assistedinject.FactoryModuleBuilder; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import org.apache.lucene.search.BooleanQuery; import org.eclipse.jgit.lib.Config; -public class LuceneIndexModule extends LifecycleModule { - private final Integer singleVersion; - private final int threads; +import java.util.Collection; +import java.util.Map; - public LuceneIndexModule() { - this(null, 0); +public class LuceneIndexModule extends LifecycleModule { + private static final String SINGLE_VERSIONS = + "LuceneIndexModule/SingleVersions"; + + public static LuceneIndexModule singleVersionAllLatest(int threads) { + return new LuceneIndexModule(ImmutableMap. of(), threads); } - public LuceneIndexModule(Integer singleVersion, int threads) { - this.singleVersion = singleVersion; + public static LuceneIndexModule singleVersionWithExplicitVersions( + Map versions, int threads) { + return new LuceneIndexModule(versions, threads); + } + + public static LuceneIndexModule latestVersionWithOnlineUpgrade() { + return new LuceneIndexModule(null, 0); + } + + static boolean isInMemoryTest(Config cfg) { + return cfg.getBoolean("index", "lucene", "testInmemory", false); + } + + private final int threads; + private final Map singleVersions; + + private LuceneIndexModule(Map singleVersions, int threads) { + this.singleVersions = singleVersions; this.threads = threads; } @Override protected void configure() { - factory(LuceneChangeIndex.Factory.class); - factory(OnlineReindexer.Factory.class); + install( + new FactoryModuleBuilder() + .implement(ChangeIndex.class, LuceneChangeIndex.class) + .build(ChangeIndex.Factory.class)); + install(new IndexModule(threads)); - if (singleVersion == null) { + if (singleVersions == null) { install(new MultiVersionModule()); } else { install(new SingleVersionModule()); @@ -57,6 +86,8 @@ public class LuceneIndexModule extends LifecycleModule { @Provides @Singleton IndexConfig getIndexConfig(@GerritServerConfig Config cfg) { + BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms", + BooleanQuery.getMaxClauseCount())); return IndexConfig.fromConfig(cfg); } @@ -71,34 +102,48 @@ public class LuceneIndexModule extends LifecycleModule { @Override public void configure() { listener().to(SingleVersionListener.class); - } - - @Provides - @Singleton - LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory) { - Schema schema = singleVersion != null - ? ChangeSchemas.get(singleVersion) - : ChangeSchemas.getLatest(); - return factory.create(schema); + bind(new TypeLiteral>() {}) + .annotatedWith(Names.named(SINGLE_VERSIONS)) + .toInstance(singleVersions); } } @Singleton static class SingleVersionListener implements LifecycleListener { - private final ChangeIndexCollection indexes; - private final LuceneChangeIndex index; + private final Collection> defs; + private final Map singleVersions; @Inject - SingleVersionListener(ChangeIndexCollection indexes, - LuceneChangeIndex index) { - this.indexes = indexes; - this.index = index; + SingleVersionListener( + Collection> defs, + @Named(SINGLE_VERSIONS) Map singleVersions) { + this.defs = defs; + this.singleVersions = singleVersions; } @Override public void start() { - indexes.setSearchIndex(index); - indexes.addWriteIndex(index); + for (IndexDefinition def : defs) { + start(def); + } + } + + private > void start( + IndexDefinition def) { + Schema schema; + Integer v = singleVersions.get(def.getName()); + if (v == null) { + schema = def.getLatest(); + } else { + schema = def.getSchemas().get(v); + if (schema == null) { + throw new ProvisionException(String.format( + "Unrecognized %s schema version: %s", def.getName(), v)); + } + } + I index = def.getIndexFactory().create(schema); + def.getIndexCollection().setSearchIndex(index); + def.getIndexCollection().addWriteIndex(index); } @Override diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java index 801b0eaef1..9da134555e 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java @@ -20,11 +20,14 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.OnlineReindexer; import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.change.ChangeIndexCollection; -import com.google.gerrit.server.index.change.ChangeSchemas; +import com.google.gerrit.server.index.change.ChangeIndexDefintion; import com.google.gerrit.server.query.change.ChangeData; import com.google.inject.Inject; import com.google.inject.ProvisionException; @@ -50,7 +53,7 @@ public class LuceneVersionManager implements LifecycleListener { private static final Logger log = LoggerFactory .getLogger(LuceneVersionManager.class); - private static final String CHANGES_PREFIX = "changes_"; + static final String CHANGES_PREFIX = "changes_"; private static class Version { private final Schema schema; @@ -68,9 +71,9 @@ public class LuceneVersionManager implements LifecycleListener { } } - static Path getDir(SitePaths sitePaths, Schema schema) { + static Path getDir(SitePaths sitePaths, String prefix, Schema schema) { return sitePaths.index_dir.resolve(String.format("%s%04d", - CHANGES_PREFIX, schema.getVersion())); + prefix, schema.getVersion())); } static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths) @@ -93,9 +96,9 @@ public class LuceneVersionManager implements LifecycleListener { private final SitePaths sitePaths; private final LuceneChangeIndex.Factory indexFactory; private final ChangeIndexCollection indexes; - private final OnlineReindexer.Factory reindexerFactory; + private final ChangeIndexDefintion changeDef; private final boolean onlineUpgrade; - private OnlineReindexer reindexer; + private OnlineReindexer reindexer; @Inject LuceneVersionManager( @@ -103,11 +106,11 @@ public class LuceneVersionManager implements LifecycleListener { SitePaths sitePaths, LuceneChangeIndex.Factory indexFactory, ChangeIndexCollection indexes, - OnlineReindexer.Factory reindexerFactory) { + ChangeIndexDefintion changeDef) { this.sitePaths = sitePaths; this.indexFactory = indexFactory; this.indexes = indexes; - this.reindexerFactory = reindexerFactory; + this.changeDef = changeDef; this.onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true); } @@ -157,7 +160,8 @@ public class LuceneVersionManager implements LifecycleListener { } markNotReady(cfg, versions.values(), write); - LuceneChangeIndex searchIndex = indexFactory.create(search.schema); + LuceneChangeIndex searchIndex = + (LuceneChangeIndex) indexFactory.create(search.schema); indexes.setSearchIndex(searchIndex); for (Version v : write) { if (v.schema != null) { @@ -171,7 +175,7 @@ public class LuceneVersionManager implements LifecycleListener { int latest = write.get(0).version; if (onlineUpgrade && latest != search.version) { - reindexer = reindexerFactory.create(latest); + reindexer = new OnlineReindexer<>(changeDef, latest); reindexer.start(); } } @@ -223,8 +227,8 @@ public class LuceneVersionManager implements LifecycleListener { private TreeMap scanVersions(Config cfg) { TreeMap versions = Maps.newTreeMap(); - for (Schema schema : ChangeSchemas.ALL.values()) { - Path p = getDir(sitePaths, schema); + for (Schema schema : changeDef.getSchemas().values()) { + Path p = getDir(sitePaths, CHANGES_PREFIX, schema); boolean isDir = Files.isDirectory(p); if (Files.exists(p) && !isDir) { log.warn("Not a directory: %s", p.toAbsolutePath()); diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java index 9897ded5d1..7e5177e692 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java @@ -14,24 +14,23 @@ package com.google.gerrit.lucene; -import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID; +import static com.google.common.base.Preconditions.checkArgument; import static org.apache.lucene.search.BooleanClause.Occur.MUST; import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT; import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; import com.google.common.collect.Lists; -import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.index.FieldType; import com.google.gerrit.server.index.IndexPredicate; import com.google.gerrit.server.index.IntegerRangePredicate; import com.google.gerrit.server.index.RegexPredicate; +import com.google.gerrit.server.index.Schema; import com.google.gerrit.server.index.TimestampRangePredicate; import com.google.gerrit.server.query.AndPredicate; import com.google.gerrit.server.query.NotPredicate; import com.google.gerrit.server.query.OrPredicate; import com.google.gerrit.server.query.Predicate; import com.google.gerrit.server.query.QueryParseException; -import com.google.gerrit.server.query.change.ChangeData; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.Term; @@ -48,23 +47,22 @@ import org.apache.lucene.util.NumericUtils; import java.util.Date; import java.util.List; -public class QueryBuilder { - - public static Term idTerm(ChangeData cd) { - return intTerm(LEGACY_ID.getName(), cd.getId().get()); - } - - public static Term idTerm(Change.Id id) { - return intTerm(LEGACY_ID.getName(), id.get()); +public class QueryBuilder { + static Term intTerm(String name, int value) { + BytesRefBuilder builder = new BytesRefBuilder(); + NumericUtils.intToPrefixCodedBytes(value, 0, builder); + return new Term(name, builder.get()); } + private final Schema schema; private final org.apache.lucene.util.QueryBuilder queryBuilder; - public QueryBuilder(Analyzer analyzer) { + public QueryBuilder(Schema schema, Analyzer analyzer) { + this.schema = schema; queryBuilder = new org.apache.lucene.util.QueryBuilder(analyzer); } - public Query toQuery(Predicate p) throws QueryParseException { + public Query toQuery(Predicate p) throws QueryParseException { if (p instanceof AndPredicate) { return and(p); } else if (p instanceof OrPredicate) { @@ -72,13 +70,13 @@ public class QueryBuilder { } else if (p instanceof NotPredicate) { return not(p); } else if (p instanceof IndexPredicate) { - return fieldQuery((IndexPredicate) p); + return fieldQuery((IndexPredicate) p); } else { throw new QueryParseException("cannot create query for index: " + p); } } - private Query or(Predicate p) + private Query or(Predicate p) throws QueryParseException { try { BooleanQuery.Builder q = new BooleanQuery.Builder(); @@ -91,17 +89,17 @@ public class QueryBuilder { } } - private Query and(Predicate p) + private Query and(Predicate p) throws QueryParseException { try { BooleanQuery.Builder b = new BooleanQuery.Builder(); List not = Lists.newArrayListWithCapacity(p.getChildCount()); for (int i = 0; i < p.getChildCount(); i++) { - Predicate c = p.getChild(i); + Predicate c = p.getChild(i); if (c instanceof NotPredicate) { - Predicate n = c.getChild(0); + Predicate n = c.getChild(0); if (n instanceof TimestampRangePredicate) { - b.add(notTimestamp((TimestampRangePredicate) n), MUST); + b.add(notTimestamp((TimestampRangePredicate) n), MUST); } else { not.add(toQuery(n)); } @@ -118,11 +116,11 @@ public class QueryBuilder { } } - private Query not(Predicate p) + private Query not(Predicate p) throws QueryParseException { - Predicate n = p.getChild(0); + Predicate n = p.getChild(0); if (n instanceof TimestampRangePredicate) { - return notTimestamp((TimestampRangePredicate) n); + return notTimestamp((TimestampRangePredicate) n); } // Lucene does not support negation, start with all and subtract. @@ -132,8 +130,11 @@ public class QueryBuilder { .build(); } - private Query fieldQuery(IndexPredicate p) + private Query fieldQuery(IndexPredicate p) throws QueryParseException { + checkArgument(schema.hasField(p.getField()), + "field not in schema v%s: %s", schema.getVersion(), + p.getField().getName()); if (p.getType() == FieldType.INTEGER) { return intQuery(p); } else if (p.getType() == FieldType.INTEGER_RANGE) { @@ -151,13 +152,7 @@ public class QueryBuilder { } } - private static Term intTerm(String name, int value) { - BytesRefBuilder builder = new BytesRefBuilder(); - NumericUtils.intToPrefixCodedBytes(value, 0, builder); - return new Term(name, builder.get()); - } - - private Query intQuery(IndexPredicate p) + private Query intQuery(IndexPredicate p) throws QueryParseException { int value; try { @@ -170,11 +165,11 @@ public class QueryBuilder { return new TermQuery(intTerm(p.getField().getName(), value)); } - private Query intRangeQuery(IndexPredicate p) + private Query intRangeQuery(IndexPredicate p) throws QueryParseException { if (p instanceof IntegerRangePredicate) { - IntegerRangePredicate r = - (IntegerRangePredicate) p; + IntegerRangePredicate r = + (IntegerRangePredicate) p; int minimum = r.getMinimumValue(); int maximum = r.getMaximumValue(); if (minimum == maximum) { @@ -192,11 +187,11 @@ public class QueryBuilder { throw new QueryParseException("not an integer range: " + p); } - private Query timestampQuery(IndexPredicate p) + private Query timestampQuery(IndexPredicate p) throws QueryParseException { if (p instanceof TimestampRangePredicate) { - TimestampRangePredicate r = - (TimestampRangePredicate) p; + TimestampRangePredicate r = + (TimestampRangePredicate) p; return NumericRangeQuery.newLongRange( r.getField().getName(), r.getMinTimestamp().getTime(), @@ -206,7 +201,7 @@ public class QueryBuilder { throw new QueryParseException("not a timestamp: " + p); } - private Query notTimestamp(TimestampRangePredicate r) + private Query notTimestamp(TimestampRangePredicate r) throws QueryParseException { if (r.getMinTimestamp().getTime() == 0) { return NumericRangeQuery.newLongRange( @@ -218,7 +213,7 @@ public class QueryBuilder { throw new QueryParseException("cannot negate: " + r); } - private Query exactQuery(IndexPredicate p) { + private Query exactQuery(IndexPredicate p) { if (p instanceof RegexPredicate) { return regexQuery(p); } else { @@ -226,7 +221,7 @@ public class QueryBuilder { } } - private Query regexQuery(IndexPredicate p) { + private Query regexQuery(IndexPredicate p) { String re = p.getValue(); if (re.startsWith("^")) { re = re.substring(1); @@ -237,11 +232,11 @@ public class QueryBuilder { return new RegexpQuery(new Term(p.getField().getName(), re)); } - private Query prefixQuery(IndexPredicate p) { + private Query prefixQuery(IndexPredicate p) { return new PrefixQuery(new Term(p.getField().getName(), p.getValue())); } - private Query fullTextQuery(IndexPredicate p) + private Query fullTextQuery(IndexPredicate p) throws QueryParseException { String value = p.getValue(); if (value == null) { diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java index 1edf4a1ae3..08b85c5e56 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java @@ -403,7 +403,9 @@ public class Daemon extends SiteProgram { } switch (indexType) { case LUCENE: - return luceneModule != null ? luceneModule : new LuceneIndexModule(); + return luceneModule != null + ? luceneModule + : LuceneIndexModule.latestVersionWithOnlineUpgrade(); default: throw new IllegalStateException("unsupported index.type = " + indexType); } diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java index 06e6003c2f..48b8b5e75b 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java @@ -31,9 +31,10 @@ import com.google.gerrit.server.git.ScanningChangeCacheImpl; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.SiteIndexer; +import com.google.gerrit.server.index.change.AllChangesIndexer; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.change.ChangeIndexCollection; -import com.google.gerrit.server.index.change.ChangeSchemas; +import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; import com.google.gerrit.server.notedb.ChangeNotes; import com.google.gerrit.server.project.ProjectCache; import com.google.inject.Injector; @@ -47,7 +48,9 @@ import org.eclipse.jgit.lib.TextProgressMonitor; import org.eclipse.jgit.util.io.NullOutputStream; import org.kohsuke.args4j.Option; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -55,9 +58,9 @@ public class Reindex extends SiteProgram { @Option(name = "--threads", usage = "Number of threads to use for indexing") private int threads = Runtime.getRuntime().availableProcessors(); - @Option(name = "--schema-version", - usage = "Schema version to reindex; default is most recent version") - private Integer version; + @Option(name = "--changes-schema-version", + usage = "Schema version to reindex, for changes; default is most recent version") + private Integer changesVersion; @Option(name = "--verbose", usage = "Output debug information for each change") private boolean verbose; @@ -82,9 +85,6 @@ public class Reindex extends SiteProgram { checkNotSlaveMode(); disableLuceneAutomaticCommit(); disableChangeCache(); - if (version == null) { - version = ChangeSchemas.getLatest().getVersion(); - } LifecycleManager dbManager = new LifecycleManager(); dbManager.add(dbInjector); dbManager.start(); @@ -120,11 +120,16 @@ public class Reindex extends SiteProgram { } private Injector createSysInjector() { + Map versions = new HashMap<>(); + if (changesVersion != null) { + versions.put(ChangeSchemaDefinitions.INSTANCE.getName(), changesVersion); + } List modules = Lists.newArrayList(); Module changeIndexModule; switch (IndexModule.getIndexType(dbInjector)) { case LUCENE: - changeIndexModule = new LuceneIndexModule(version, threads); + changeIndexModule = LuceneIndexModule.singleVersionWithExplicitVersions( + versions, threads); break; default: throw new IllegalStateException("unsupported index.type"); @@ -164,9 +169,9 @@ public class Reindex extends SiteProgram { } pm.endTask(); - SiteIndexer batchIndexer = - sysInjector.getInstance(SiteIndexer.class); - SiteIndexer.Result result = batchIndexer.setNumChanges(changeCount) + AllChangesIndexer batchIndexer = + sysInjector.getInstance(AllChangesIndexer.class); + SiteIndexer.Result result = batchIndexer.setTotalWork(changeCount) .setProgressOut(System.err) .setVerboseOut(verbose ? System.out : NullOutputStream.INSTANCE) .indexAll(index, projects); diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java index 53bb349e4e..0883ac6f58 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java @@ -15,14 +15,15 @@ package com.google.gerrit.pgm.init; import com.google.common.collect.Iterables; -import com.google.gerrit.lucene.LuceneChangeIndex; +import com.google.gerrit.lucene.AbstractLuceneIndex; import com.google.gerrit.pgm.init.api.ConsoleUI; import com.google.gerrit.pgm.init.api.InitFlags; import com.google.gerrit.pgm.init.api.InitStep; import com.google.gerrit.pgm.init.api.Section; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.IndexModule.IndexType; -import com.google.gerrit.server.index.change.ChangeSchemas; +import com.google.gerrit.server.index.SchemaDefinitions; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -57,8 +58,11 @@ class InitIndex implements InitStep { ui.header("Index"); IndexType type = index.select("Type", "type", IndexType.LUCENE); - LuceneChangeIndex.setReady( - site, ChangeSchemas.getLatest().getVersion(), true); + for (SchemaDefinitions def : IndexModule.ALL_SCHEMA_DEFS) { + // TODO(dborowitz): Totally broken for non-change indexes. + AbstractLuceneIndex.setReady( + site, def.getLatest().getVersion(), true); + } if ((site.isNew || isEmptySite()) && type == IndexType.LUCENE) { } else { final String message = String.format( diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java index b4188a741e..678b8df5b8 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java @@ -14,15 +14,24 @@ package com.google.gerrit.server.index; +import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.change.DummyChangeIndex; +import com.google.gerrit.server.query.change.ChangeData; import com.google.inject.AbstractModule; public class DummyIndexModule extends AbstractModule { + private static class DummyChangeIndexFactory implements ChangeIndex.Factory { + @Override + public ChangeIndex create(Schema schema) { + throw new UnsupportedOperationException(); + } + } @Override protected void configure() { install(new IndexModule(1)); bind(IndexConfig.class).toInstance(IndexConfig.createDefault()); bind(Index.class).toInstance(new DummyChangeIndex()); + bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory()); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexDefinition.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexDefinition.java new file mode 100644 index 0000000000..629dff8804 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexDefinition.java @@ -0,0 +1,71 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index; + +import com.google.common.collect.ImmutableSortedMap; + +/** + * Definition of an index over a Gerrit data type. + *

+ * An index includes a set of schema definitions along with the + * specific implementations used to query the secondary index implementation in + * a running server. If you are just interested in the static definition of one + * or more schemas, see the implementations of {@link SchemaDefinitions}. + */ +public abstract class IndexDefinition> { + public interface IndexFactory> { + I create(Schema schema); + } + + private final SchemaDefinitions schemaDefs; + private final IndexCollection indexCollection; + private final IndexFactory indexFactory; + private final SiteIndexer siteIndexer; + + protected IndexDefinition( + SchemaDefinitions schemaDefs, + IndexCollection indexCollection, + IndexFactory indexFactory, + SiteIndexer siteIndexer) { + this.schemaDefs = schemaDefs; + this.indexCollection = indexCollection; + this.indexFactory = indexFactory; + this.siteIndexer = siteIndexer; + } + + public final String getName() { + return schemaDefs.getName(); + } + + public final ImmutableSortedMap> getSchemas() { + return schemaDefs.getSchemas(); + } + + public final Schema getLatest() { + return schemaDefs.getLatest(); + } + + public final IndexCollection getIndexCollection() { + return indexCollection; + } + + public final IndexFactory getIndexFactory() { + return indexFactory; + } + + public final SiteIndexer getSiteIndexer() { + return siteIndexer; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java index b8ce16094f..84cadcc340 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java @@ -17,21 +17,31 @@ package com.google.gerrit.server.index; import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH; import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE; +import com.google.common.base.Function; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.gerrit.lifecycle.LifecycleModule; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.index.change.ChangeIndexCollection; +import com.google.gerrit.server.index.change.ChangeIndexDefintion; import com.google.gerrit.server.index.change.ChangeIndexer; +import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; import com.google.gerrit.server.index.change.IndexRewriter; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Provides; +import com.google.inject.ProvisionException; import com.google.inject.Singleton; import org.eclipse.jgit.lib.Config; +import java.util.Collection; +import java.util.Set; + /** * Module for non-indexer-specific secondary index setup. *

@@ -43,6 +53,10 @@ public class IndexModule extends LifecycleModule { LUCENE } + public static final ImmutableCollection> ALL_SCHEMA_DEFS = + ImmutableList.> of( + ChangeSchemaDefinitions.INSTANCE); + /** Type of secondary index. */ public static IndexType getIndexType(Injector injector) { Config cfg = injector.getInstance( @@ -74,6 +88,33 @@ public class IndexModule extends LifecycleModule { factory(ChangeIndexer.Factory.class); } + @Provides + Collection> getIndexDefinitions( + ChangeIndexDefintion changes) { + Collection> result = + ImmutableList.> of(changes); + Set expected = FluentIterable.from(ALL_SCHEMA_DEFS) + .transform(new Function, String>() { + @Override + public String apply(SchemaDefinitions in) { + return in.getName(); + } + }).toSet(); + Set actual = FluentIterable.from(result) + .transform(new Function, String>() { + @Override + public String apply(IndexDefinition in) { + return in.getName(); + } + }).toSet(); + if (!expected.equals(actual)) { + throw new ProvisionException( + "need index definitions for all schemas: " + + expected + " != " + actual); + } + return result; + } + @Provides @Singleton ChangeIndexer getChangeIndexer( diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/OnlineReindexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/OnlineReindexer.java similarity index 70% rename from gerrit-lucene/src/main/java/com/google/gerrit/lucene/OnlineReindexer.java rename to gerrit-server/src/main/java/com/google/gerrit/server/index/OnlineReindexer.java index 89aa57cabc..133d78b373 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/OnlineReindexer.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/OnlineReindexer.java @@ -12,18 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.google.gerrit.lucene; +package com.google.gerrit.server.index; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.Lists; -import com.google.gerrit.server.index.Index; -import com.google.gerrit.server.index.SiteIndexer; -import com.google.gerrit.server.index.change.ChangeIndex; -import com.google.gerrit.server.index.change.ChangeIndexCollection; -import com.google.gerrit.server.project.ProjectCache; -import com.google.inject.Inject; -import com.google.inject.assistedinject.Assisted; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,30 +25,21 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -public class OnlineReindexer { +public class OnlineReindexer> { private static final Logger log = LoggerFactory .getLogger(OnlineReindexer.class); - public interface Factory { - OnlineReindexer create(int version); - } - - private final ChangeIndexCollection indexes; - private final SiteIndexer batchIndexer; - private final ProjectCache projectCache; + private final IndexCollection indexes; + private final SiteIndexer batchIndexer; private final int version; - private ChangeIndex index; + private I index; private final AtomicBoolean running = new AtomicBoolean(); - @Inject - OnlineReindexer( - ChangeIndexCollection indexes, - SiteIndexer batchIndexer, - ProjectCache projectCache, - @Assisted int version) { - this.indexes = indexes; - this.batchIndexer = batchIndexer; - this.projectCache = projectCache; + public OnlineReindexer( + IndexDefinition def, + int version) { + this.indexes = def.getIndexCollection(); + this.batchIndexer = def.getSiteIndexer(); this.version = version; } @@ -94,8 +78,7 @@ public class OnlineReindexer { "not an active write schema version: %s", version); log.info("Starting online reindex from schema version {} to {}", version(indexes.getSearchIndex()), version(index)); - SiteIndexer.Result result = - batchIndexer.indexAll(index, projectCache.all()); + SiteIndexer.Result result = batchIndexer.indexAll(index); if (!result.success()) { log.error("Online reindex of schema version {} failed. Successfully" + " indexed {} changes, failed to index {} changes", @@ -106,7 +89,7 @@ public class OnlineReindexer { activateIndex(); } - void activateIndex() { + public void activateIndex() { indexes.setSearchIndex(index); log.info("Using schema version {}", version(index)); try { @@ -115,13 +98,13 @@ public class OnlineReindexer { log.warn("Error activating new schema version {}", version(index)); } - List toRemove = Lists.newArrayListWithExpectedSize(1); - for (ChangeIndex i : indexes.getWriteIndexes()) { + List toRemove = Lists.newArrayListWithExpectedSize(1); + for (I i : indexes.getWriteIndexes()) { if (version(i) != version(index)) { toRemove.add(i); } } - for (ChangeIndex i : toRemove) { + for (I i : toRemove) { try { i.markReady(false); indexes.removeWriteIndex(version(i)); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaDefinitions.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaDefinitions.java new file mode 100644 index 0000000000..f9a799e253 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaDefinitions.java @@ -0,0 +1,58 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableSortedMap; + +/** + * Definitions of the various schema versions over a given Gerrit data type. + *

+ * A schema is a description of the fields that are indexed over the + * given data type. This class contains all the versions of a schema defined + * over its data type, exposed as a map of version number to schema definition. + * If you are interested in the classes responsible for backend-specific runtime + * implementations, see the implementations of {@link IndexDefinition}. + */ +public abstract class SchemaDefinitions { + private final String name; + private final ImmutableSortedMap> schemas; + + protected SchemaDefinitions(String name, Class valueClass) { + this.name = checkNotNull(name); + this.schemas = SchemaUtil.schemasFromClass(getClass(), valueClass); + } + + public final String getName() { + return name; + } + + public final ImmutableSortedMap> getSchemas() { + return schemas; + } + + public final Schema get(int version) { + Schema schema = schemas.get(version); + checkArgument(schema != null, + "Unrecognized %s schema version: %s", name, version); + return schema; + } + + public final Schema getLatest() { + return schemas.lastEntry().getValue(); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaUtil.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaUtil.java index 404357d99f..53ff0e3d90 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaUtil.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaUtil.java @@ -19,8 +19,8 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.common.base.CharMatcher; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -30,16 +30,16 @@ import org.eclipse.jgit.lib.PersonIdent; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; public class SchemaUtil { - public static ImmutableMap> schemasFromClass( + public static ImmutableSortedMap> schemasFromClass( Class schemasClass, Class valueClass) { - Map> schemas = Maps.newTreeMap(); + Map> schemas = Maps.newHashMap(); for (Field f : schemasClass.getDeclaredFields()) { if (Modifier.isStatic(f.getModifiers()) && Modifier.isFinal(f.getModifiers()) @@ -65,7 +65,7 @@ public class SchemaUtil { if (schemas.isEmpty()) { throw new ExceptionInInitializerError("no ChangeSchemas found"); } - return ImmutableMap.copyOf(schemas); + return ImmutableSortedMap.copyOf(schemas); } public static Schema schema(Collection> fields) { @@ -81,13 +81,28 @@ public class SchemaUtil { if (person == null) { return ImmutableSet.of(); } - HashSet parts = Sets.newHashSet(); - String email = person.getEmailAddress().toLowerCase(); - parts.add(email); - parts.addAll(Arrays.asList(email.split("@"))); + return getPersonParts( + person.getName(), + Collections.singleton(person.getEmailAddress())); + } + + public static Set getPersonParts(String name, + Iterable emails) { + Splitter at = Splitter.on('@'); Splitter s = Splitter.on(CharMatcher.anyOf("@.- ")).omitEmptyStrings(); - Iterables.addAll(parts, s.split(email)); - Iterables.addAll(parts, s.split(person.getName().toLowerCase())); + HashSet parts = Sets.newHashSet(); + for (String email : emails) { + if (email == null) { + continue; + } + String lowerEmail = email.toLowerCase(); + parts.add(lowerEmail); + Iterables.addAll(parts, at.split(lowerEmail)); + Iterables.addAll(parts, s.split(lowerEmail)); + } + if (name != null) { + Iterables.addAll(parts, s.split(name.toLowerCase())); + } return parts; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/SiteIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/SiteIndexer.java index a60c2a839e..46a2b7dbf8 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/SiteIndexer.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/SiteIndexer.java @@ -1,4 +1,4 @@ -// Copyright (C) 2013 The Android Open Source Project +// Copyright (C) 2016 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,81 +14,18 @@ package com.google.gerrit.server.index; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH; -import static org.eclipse.jgit.lib.RefDatabase.ALL; - import com.google.common.base.Stopwatch; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.reviewdb.server.ReviewDb; -import com.google.gerrit.server.config.GerritServerConfig; -import com.google.gerrit.server.git.GitRepositoryManager; -import com.google.gerrit.server.git.MergeUtil; -import com.google.gerrit.server.git.MultiProgressMonitor; -import com.google.gerrit.server.git.MultiProgressMonitor.Task; -import com.google.gerrit.server.index.change.ChangeIndex; -import com.google.gerrit.server.index.change.ChangeIndexer; -import com.google.gerrit.server.notedb.ChangeNotes; -import com.google.gerrit.server.patch.PatchListLoader; -import com.google.gerrit.server.query.change.ChangeData; -import com.google.gwtorm.server.SchemaFactory; -import com.google.inject.Inject; -import org.eclipse.jgit.diff.DiffEntry; -import org.eclipse.jgit.diff.DiffFormatter; -import org.eclipse.jgit.errors.RepositoryNotFoundException; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.merge.ThreeWayMergeStrategy; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.io.DisabledOutputStream; -import org.eclipse.jgit.util.io.NullOutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -public class SiteIndexer { - private static final Logger log = - LoggerFactory.getLogger(SiteIndexer.class); - - public static class Result { +public interface SiteIndexer> { + public class Result { private final long elapsedNanos; private final boolean success; private final int done; private final int failed; - private Result(Stopwatch sw, boolean success, int done, int failed) { + public Result(Stopwatch sw, boolean success, int done, int failed) { this.elapsedNanos = sw.elapsed(TimeUnit.NANOSECONDS); this.success = success; this.done = done; @@ -112,313 +49,5 @@ public class SiteIndexer { } } - private final SchemaFactory schemaFactory; - private final ChangeData.Factory changeDataFactory; - private final GitRepositoryManager repoManager; - private final ListeningExecutorService executor; - private final ChangeIndexer.Factory indexerFactory; - private final ChangeNotes.Factory notesFactory; - private final ThreeWayMergeStrategy mergeStrategy; - - private int numChanges = -1; - private OutputStream progressOut = NullOutputStream.INSTANCE; - private PrintWriter verboseWriter = - new PrintWriter(NullOutputStream.INSTANCE); - - @Inject - SiteIndexer(SchemaFactory schemaFactory, - ChangeData.Factory changeDataFactory, - GitRepositoryManager repoManager, - @IndexExecutor(BATCH) ListeningExecutorService executor, - ChangeIndexer.Factory indexerFactory, - ChangeNotes.Factory notesFactory, - @GerritServerConfig Config config) { - this.schemaFactory = schemaFactory; - this.changeDataFactory = changeDataFactory; - this.repoManager = repoManager; - this.executor = executor; - this.indexerFactory = indexerFactory; - this.notesFactory = notesFactory; - this.mergeStrategy = MergeUtil.getMergeStrategy(config); - } - - public SiteIndexer setNumChanges(int num) { - numChanges = num; - return this; - } - - public SiteIndexer setProgressOut(OutputStream out) { - progressOut = checkNotNull(out); - return this; - } - - public SiteIndexer setVerboseOut(OutputStream out) { - verboseWriter = new PrintWriter(checkNotNull(out)); - return this; - } - - public Result indexAll(ChangeIndex index, - Iterable projects) { - Stopwatch sw = Stopwatch.createStarted(); - final MultiProgressMonitor mpm = - new MultiProgressMonitor(progressOut, "Reindexing changes"); - final Task projTask = mpm.beginSubTask("projects", - (projects instanceof Collection) - ? ((Collection) projects).size() - : MultiProgressMonitor.UNKNOWN); - final Task doneTask = mpm.beginSubTask(null, - numChanges >= 0 ? numChanges : MultiProgressMonitor.UNKNOWN); - final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN); - - final List> futures = Lists.newArrayList(); - final AtomicBoolean ok = new AtomicBoolean(true); - - for (final Project.NameKey project : projects) { - final ListenableFuture future = executor.submit(reindexProject( - indexerFactory.create(executor, index), project, doneTask, failedTask, - verboseWriter)); - futures.add(future); - future.addListener(new Runnable() { - @Override - public void run() { - try { - future.get(); - } catch (ExecutionException | InterruptedException e) { - fail(project, e); - } catch (RuntimeException e) { - failAndThrow(project, e); - } catch (Error e) { - // Can't join with RuntimeException because "RuntimeException | - // Error" becomes Throwable, which messes with signatures. - failAndThrow(project, e); - } finally { - projTask.update(1); - } - } - - private void fail(Project.NameKey project, Throwable t) { - log.error("Failed to index project " + project, t); - ok.set(false); - } - - private void failAndThrow(Project.NameKey project, RuntimeException e) { - fail(project, e); - throw e; - } - - private void failAndThrow(Project.NameKey project, Error e) { - fail(project, e); - throw e; - } - }, MoreExecutors.directExecutor()); - } - - try { - mpm.waitFor(Futures.transformAsync(Futures.successfulAsList(futures), - new AsyncFunction, Void>() { - @Override - public ListenableFuture apply(List input) { - mpm.end(); - return Futures.immediateFuture(null); - } - })); - } catch (ExecutionException e) { - log.error("Error in batch indexer", e); - ok.set(false); - } - // If too many changes failed, maybe there was a bug in the indexer. Don't - // trust the results. This is not an exact percentage since we bump the same - // failure counter if a project can't be read, but close enough. - int nFailed = failedTask.getCount(); - int nTotal = nFailed + doneTask.getCount(); - double pctFailed = ((double) nFailed) / nTotal * 100; - if (pctFailed > 10) { - log.error("Failed {}/{} changes ({}%); not marking new index as ready", - nFailed, nTotal, Math.round(pctFailed)); - ok.set(false); - } - return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount()); - } - - private Callable reindexProject(final ChangeIndexer indexer, - final Project.NameKey project, final Task done, final Task failed, - final PrintWriter verboseWriter) { - return new Callable() { - @Override - public Void call() throws Exception { - Multimap byId = ArrayListMultimap.create(); - // TODO(dborowitz): Opening all repositories in a live server may be - // wasteful; see if we can determine which ones it is safe to close - // with RepositoryCache.close(repo). - try (Repository repo = repoManager.openRepository(project); - ReviewDb db = schemaFactory.open()) { - Map refs = repo.getRefDatabase().getRefs(ALL); - for (ChangeNotes cn : notesFactory.scan(repo, db, project)) { - Ref r = refs.get(cn.getChange().currentPatchSetId().toRefName()); - if (r != null) { - byId.put(r.getObjectId(), changeDataFactory.create(db, cn)); - } - } - new ProjectIndexer(indexer, - mergeStrategy, - byId, - repo, - done, - failed, - verboseWriter).call(); - } catch (RepositoryNotFoundException rnfe) { - log.error(rnfe.getMessage()); - } - return null; - } - - @Override - public String toString() { - return "Index all changes of project " + project.get(); - } - }; - } - - private static class ProjectIndexer implements Callable { - private final ChangeIndexer indexer; - private final ThreeWayMergeStrategy mergeStrategy; - private final Multimap byId; - private final ProgressMonitor done; - private final ProgressMonitor failed; - private final PrintWriter verboseWriter; - private final Repository repo; - private RevWalk walk; - - private ProjectIndexer(ChangeIndexer indexer, - ThreeWayMergeStrategy mergeStrategy, - Multimap changesByCommitId, - Repository repo, - ProgressMonitor done, - ProgressMonitor failed, - PrintWriter verboseWriter) { - this.indexer = indexer; - this.mergeStrategy = mergeStrategy; - this.byId = changesByCommitId; - this.repo = repo; - this.done = done; - this.failed = failed; - this.verboseWriter = verboseWriter; - } - - @Override - public Void call() throws Exception { - walk = new RevWalk(repo); - try { - // Walk only refs first to cover as many changes as we can without having - // to mark every single change. - for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) { - RevObject o = walk.parseAny(ref.getObjectId()); - if (o instanceof RevCommit) { - walk.markStart((RevCommit) o); - } - } - - RevCommit bCommit; - while ((bCommit = walk.next()) != null && !byId.isEmpty()) { - if (byId.containsKey(bCommit)) { - getPathsAndIndex(bCommit); - byId.removeAll(bCommit); - } - } - - for (ObjectId id : byId.keySet()) { - getPathsAndIndex(id); - } - } finally { - walk.close(); - } - return null; - } - - private void getPathsAndIndex(ObjectId b) throws Exception { - List cds = Lists.newArrayList(byId.get(b)); - try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) { - RevCommit bCommit = walk.parseCommit(b); - RevTree bTree = bCommit.getTree(); - RevTree aTree = aFor(bCommit, walk); - df.setRepository(repo); - if (!cds.isEmpty()) { - List paths = (aTree != null) - ? getPaths(df.scan(aTree, bTree)) - : Collections.emptyList(); - Iterator cdit = cds.iterator(); - for (ChangeData cd ; cdit.hasNext(); cdit.remove()) { - cd = cdit.next(); - try { - cd.setCurrentFilePaths(paths); - indexer.index(cd); - done.update(1); - if (verboseWriter != null) { - verboseWriter.println("Reindexed change " + cd.getId()); - } - } catch (Exception e) { - fail("Failed to index change " + cd.getId(), true, e); - } - } - } - } catch (Exception e) { - fail("Failed to index commit " + b.name(), false, e); - for (ChangeData cd : cds) { - fail("Failed to index change " + cd.getId(), true, null); - } - } - } - - private List getPaths(List filenames) { - Set paths = Sets.newTreeSet(); - for (DiffEntry e : filenames) { - if (e.getOldPath() != null) { - paths.add(e.getOldPath()); - } - if (e.getNewPath() != null) { - paths.add(e.getNewPath()); - } - } - return ImmutableList.copyOf(paths); - } - - private RevTree aFor(RevCommit b, RevWalk walk) throws IOException { - switch (b.getParentCount()) { - case 0: - return walk.parseTree(emptyTree()); - case 1: - RevCommit a = b.getParent(0); - walk.parseBody(a); - return walk.parseTree(a.getTree()); - case 2: - return PatchListLoader.automerge(repo, walk, b, mergeStrategy); - default: - return null; - } - } - - private ObjectId emptyTree() throws IOException { - try (ObjectInserter oi = repo.newObjectInserter()) { - ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {}); - oi.flush(); - return id; - } - } - - private void fail(String error, boolean failed, Exception e) { - if (failed) { - this.failed.update(1); - } - - if (e != null) { - log.warn(error, e); - } else { - log.warn(error); - } - - if (verboseWriter != null) { - verboseWriter.println(error); - } - } - } + Result indexAll(I index); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountField.java new file mode 100644 index 0000000000..6da7eef06d --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountField.java @@ -0,0 +1,140 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.account; + +import com.google.common.base.Function; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.Iterables; +import com.google.gerrit.reviewdb.client.AccountExternalId; +import com.google.gerrit.server.account.AccountState; +import com.google.gerrit.server.index.FieldDef; +import com.google.gerrit.server.index.FieldType; +import com.google.gerrit.server.index.SchemaUtil; + +import java.sql.Timestamp; +import java.util.Collections; +import java.util.Set; + +/** Secondary index schemas for accounts. */ +public class AccountField { + public static final FieldDef ID = + new FieldDef.Single( + "id", FieldType.INTEGER, true) { + @Override + public Integer get(AccountState input, FillArgs args) { + return input.getAccount().getId().get(); + } + }; + + public static final FieldDef> EXTERNAL_ID = + new FieldDef.Repeatable( + "external_id", FieldType.EXACT, false) { + @Override + public Iterable get(AccountState input, FillArgs args) { + return Iterables.transform( + input.getExternalIds(), + new Function() { + @Override + public String apply(AccountExternalId in) { + return in.getKey().get(); + } + }); + } + }; + + /** Fuzzy prefix match on name and email parts. */ + public static final FieldDef> NAME_PART = + new FieldDef.Repeatable( + "name", FieldType.PREFIX, false) { + @Override + public Iterable get(AccountState input, FillArgs args) { + String fullName = input.getAccount().getFullName(); + Set parts = SchemaUtil.getPersonParts( + fullName, + Iterables.transform( + input.getExternalIds(), + new Function() { + @Override + public String apply(AccountExternalId in) { + return in.getEmailAddress(); + } + })); + + // Additional values not currently added by getPersonParts. + // TODO(dborowitz): Move to getPersonParts and remove this hack. + if (fullName != null) { + parts.add(fullName); + } + return parts; + } + }; + + public static final FieldDef ACTIVE = + new FieldDef.Single( + "inactive", FieldType.EXACT, false) { + @Override + public String get(AccountState input, FillArgs args) { + return input.getAccount().isActive() ? "1" : "0"; + } + }; + + public static final FieldDef> EMAIL = + new FieldDef.Repeatable( + "email", FieldType.PREFIX, false) { + @Override + public Iterable get(AccountState input, FillArgs args) { + return FluentIterable.from(input.getExternalIds()) + .transform( + new Function() { + @Override + public String apply(AccountExternalId in) { + return in.getEmailAddress(); + } + }) + .append( + Collections.singleton(input.getAccount().getPreferredEmail())) + .filter(Predicates.notNull()) + .transform( + new Function() { + @Override + public String apply(String in) { + return in.toLowerCase(); + } + }); + } + }; + + public static final FieldDef REGISTERED = + new FieldDef.Single( + "registered", FieldType.TIMESTAMP, false) { + @Override + public Timestamp get(AccountState input, FillArgs args) { + return input.getAccount().getRegisteredOn(); + } + }; + + public static final FieldDef USERNAME = + new FieldDef.Single( + "username", null, false) { + @Override + public String get(AccountState input, FillArgs args) { + return input.getUserName().toLowerCase(); + } + }; + + private AccountField() { + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountSchemas.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountSchemas.java new file mode 100644 index 0000000000..1c94706d0f --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountSchemas.java @@ -0,0 +1,62 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.account; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.gerrit.server.account.AccountState; +import com.google.gerrit.server.index.FieldDef; +import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.index.SchemaUtil; + +import java.util.Collection; + +public class AccountSchemas { + static final Schema V1 = schema( + AccountField.ID, + AccountField.ACTIVE, + AccountField.EMAIL, + AccountField.EXTERNAL_ID, + AccountField.NAME_PART, + AccountField.REGISTERED, + AccountField.USERNAME); + + private static Schema schema( + Collection> fields) { + return new Schema<>(ImmutableList.copyOf(fields)); + } + + @SafeVarargs + private static Schema schema( + FieldDef... fields) { + return schema(ImmutableList.copyOf(fields)); + } + + public static final ImmutableMap> ALL = + SchemaUtil.schemasFromClass(AccountSchemas.class, AccountState.class); + + public static Schema get(int version) { + Schema schema = ALL.get(version); + checkArgument(schema != null, "Unrecognized schema version: %s", version); + return schema; + } + + public static Schema getLatest() { + return Iterables.getLast(ALL.values()); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/change/AllChangesIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/AllChangesIndexer.java new file mode 100644 index 0000000000..4fe589c43e --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/AllChangesIndexer.java @@ -0,0 +1,404 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.change; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH; +import static org.eclipse.jgit.lib.RefDatabase.ALL; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.AsyncFunction; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gerrit.reviewdb.client.Change; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.server.ReviewDb; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gerrit.server.git.MergeUtil; +import com.google.gerrit.server.git.MultiProgressMonitor; +import com.google.gerrit.server.git.MultiProgressMonitor.Task; +import com.google.gerrit.server.index.IndexExecutor; +import com.google.gerrit.server.index.SiteIndexer; +import com.google.gerrit.server.notedb.ChangeNotes; +import com.google.gerrit.server.patch.PatchListLoader; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.gwtorm.server.SchemaFactory; +import com.google.inject.Inject; + +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.merge.ThreeWayMergeStrategy; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.util.io.DisabledOutputStream; +import org.eclipse.jgit.util.io.NullOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class AllChangesIndexer + implements SiteIndexer { + private static final Logger log = + LoggerFactory.getLogger(AllChangesIndexer.class); + + private final SchemaFactory schemaFactory; + private final ChangeData.Factory changeDataFactory; + private final GitRepositoryManager repoManager; + private final ListeningExecutorService executor; + private final ChangeIndexer.Factory indexerFactory; + private final ChangeNotes.Factory notesFactory; + private final ProjectCache projectCache; + private final ThreeWayMergeStrategy mergeStrategy; + + private int totalWork = -1; + private OutputStream progressOut = NullOutputStream.INSTANCE; + private PrintWriter verboseWriter = + new PrintWriter(NullOutputStream.INSTANCE); + + @Inject + AllChangesIndexer(SchemaFactory schemaFactory, + ChangeData.Factory changeDataFactory, + GitRepositoryManager repoManager, + @IndexExecutor(BATCH) ListeningExecutorService executor, + ChangeIndexer.Factory indexerFactory, + ChangeNotes.Factory notesFactory, + @GerritServerConfig Config config, + ProjectCache projectCache) { + this.schemaFactory = schemaFactory; + this.changeDataFactory = changeDataFactory; + this.repoManager = repoManager; + this.executor = executor; + this.indexerFactory = indexerFactory; + this.notesFactory = notesFactory; + this.projectCache = projectCache; + this.mergeStrategy = MergeUtil.getMergeStrategy(config); + } + + public AllChangesIndexer setTotalWork(int num) { + totalWork = num; + return this; + } + + public AllChangesIndexer setProgressOut(OutputStream out) { + progressOut = checkNotNull(out); + return this; + } + + public AllChangesIndexer setVerboseOut(OutputStream out) { + verboseWriter = new PrintWriter(checkNotNull(out)); + return this; + } + + @Override + public Result indexAll(ChangeIndex index) { + return indexAll(index, projectCache.all()); + } + + public SiteIndexer.Result indexAll(ChangeIndex index, + Iterable projects) { + Stopwatch sw = Stopwatch.createStarted(); + final MultiProgressMonitor mpm = + new MultiProgressMonitor(progressOut, "Reindexing changes"); + final Task projTask = mpm.beginSubTask("projects", + (projects instanceof Collection) + ? ((Collection) projects).size() + : MultiProgressMonitor.UNKNOWN); + final Task doneTask = mpm.beginSubTask(null, + totalWork >= 0 ? totalWork : MultiProgressMonitor.UNKNOWN); + final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN); + + final List> futures = Lists.newArrayList(); + final AtomicBoolean ok = new AtomicBoolean(true); + + for (final Project.NameKey project : projects) { + final ListenableFuture future = executor.submit(reindexProject( + indexerFactory.create(executor, index), project, doneTask, failedTask, + verboseWriter)); + futures.add(future); + future.addListener(new Runnable() { + @Override + public void run() { + try { + future.get(); + } catch (ExecutionException | InterruptedException e) { + fail(project, e); + } catch (RuntimeException e) { + failAndThrow(project, e); + } catch (Error e) { + // Can't join with RuntimeException because "RuntimeException | + // Error" becomes Throwable, which messes with signatures. + failAndThrow(project, e); + } finally { + projTask.update(1); + } + } + + private void fail(Project.NameKey project, Throwable t) { + log.error("Failed to index project " + project, t); + ok.set(false); + } + + private void failAndThrow(Project.NameKey project, RuntimeException e) { + fail(project, e); + throw e; + } + + private void failAndThrow(Project.NameKey project, Error e) { + fail(project, e); + throw e; + } + }, MoreExecutors.directExecutor()); + } + + try { + mpm.waitFor(Futures.transformAsync(Futures.successfulAsList(futures), + new AsyncFunction, Void>() { + @Override + public ListenableFuture apply(List input) { + mpm.end(); + return Futures.immediateFuture(null); + } + })); + } catch (ExecutionException e) { + log.error("Error in batch indexer", e); + ok.set(false); + } + // If too many changes failed, maybe there was a bug in the indexer. Don't + // trust the results. This is not an exact percentage since we bump the same + // failure counter if a project can't be read, but close enough. + int nFailed = failedTask.getCount(); + int nTotal = nFailed + doneTask.getCount(); + double pctFailed = ((double) nFailed) / nTotal * 100; + if (pctFailed > 10) { + log.error("Failed {}/{} changes ({}%); not marking new index as ready", + nFailed, nTotal, Math.round(pctFailed)); + ok.set(false); + } + return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount()); + } + + private Callable reindexProject(final ChangeIndexer indexer, + final Project.NameKey project, final Task done, final Task failed, + final PrintWriter verboseWriter) { + return new Callable() { + @Override + public Void call() throws Exception { + Multimap byId = ArrayListMultimap.create(); + // TODO(dborowitz): Opening all repositories in a live server may be + // wasteful; see if we can determine which ones it is safe to close + // with RepositoryCache.close(repo). + try (Repository repo = repoManager.openRepository(project); + ReviewDb db = schemaFactory.open()) { + Map refs = repo.getRefDatabase().getRefs(ALL); + for (ChangeNotes cn : notesFactory.scan(repo, db, project)) { + Ref r = refs.get(cn.getChange().currentPatchSetId().toRefName()); + if (r != null) { + byId.put(r.getObjectId(), changeDataFactory.create(db, cn)); + } + } + new ProjectIndexer(indexer, + mergeStrategy, + byId, + repo, + done, + failed, + verboseWriter).call(); + } catch (RepositoryNotFoundException rnfe) { + log.error(rnfe.getMessage()); + } + return null; + } + + @Override + public String toString() { + return "Index all changes of project " + project.get(); + } + }; + } + + private static class ProjectIndexer implements Callable { + private final ChangeIndexer indexer; + private final ThreeWayMergeStrategy mergeStrategy; + private final Multimap byId; + private final ProgressMonitor done; + private final ProgressMonitor failed; + private final PrintWriter verboseWriter; + private final Repository repo; + private RevWalk walk; + + private ProjectIndexer(ChangeIndexer indexer, + ThreeWayMergeStrategy mergeStrategy, + Multimap changesByCommitId, + Repository repo, + ProgressMonitor done, + ProgressMonitor failed, + PrintWriter verboseWriter) { + this.indexer = indexer; + this.mergeStrategy = mergeStrategy; + this.byId = changesByCommitId; + this.repo = repo; + this.done = done; + this.failed = failed; + this.verboseWriter = verboseWriter; + } + + @Override + public Void call() throws Exception { + walk = new RevWalk(repo); + try { + // Walk only refs first to cover as many changes as we can without having + // to mark every single change. + for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) { + RevObject o = walk.parseAny(ref.getObjectId()); + if (o instanceof RevCommit) { + walk.markStart((RevCommit) o); + } + } + + RevCommit bCommit; + while ((bCommit = walk.next()) != null && !byId.isEmpty()) { + if (byId.containsKey(bCommit)) { + getPathsAndIndex(bCommit); + byId.removeAll(bCommit); + } + } + + for (ObjectId id : byId.keySet()) { + getPathsAndIndex(id); + } + } finally { + walk.close(); + } + return null; + } + + private void getPathsAndIndex(ObjectId b) throws Exception { + List cds = Lists.newArrayList(byId.get(b)); + try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) { + RevCommit bCommit = walk.parseCommit(b); + RevTree bTree = bCommit.getTree(); + RevTree aTree = aFor(bCommit, walk); + df.setRepository(repo); + if (!cds.isEmpty()) { + List paths = (aTree != null) + ? getPaths(df.scan(aTree, bTree)) + : Collections.emptyList(); + Iterator cdit = cds.iterator(); + for (ChangeData cd ; cdit.hasNext(); cdit.remove()) { + cd = cdit.next(); + try { + cd.setCurrentFilePaths(paths); + indexer.index(cd); + done.update(1); + if (verboseWriter != null) { + verboseWriter.println("Reindexed change " + cd.getId()); + } + } catch (Exception e) { + fail("Failed to index change " + cd.getId(), true, e); + } + } + } + } catch (Exception e) { + fail("Failed to index commit " + b.name(), false, e); + for (ChangeData cd : cds) { + fail("Failed to index change " + cd.getId(), true, null); + } + } + } + + private List getPaths(List filenames) { + Set paths = Sets.newTreeSet(); + for (DiffEntry e : filenames) { + if (e.getOldPath() != null) { + paths.add(e.getOldPath()); + } + if (e.getNewPath() != null) { + paths.add(e.getNewPath()); + } + } + return ImmutableList.copyOf(paths); + } + + private RevTree aFor(RevCommit b, RevWalk walk) throws IOException { + switch (b.getParentCount()) { + case 0: + return walk.parseTree(emptyTree()); + case 1: + RevCommit a = b.getParent(0); + walk.parseBody(a); + return walk.parseTree(a.getTree()); + case 2: + return PatchListLoader.automerge(repo, walk, b, mergeStrategy); + default: + return null; + } + } + + private ObjectId emptyTree() throws IOException { + try (ObjectInserter oi = repo.newObjectInserter()) { + ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {}); + oi.flush(); + return id; + } + } + + private void fail(String error, boolean failed, Exception e) { + if (failed) { + this.failed.update(1); + } + + if (e != null) { + log.warn(error, e); + } else { + log.warn(error); + } + + if (verboseWriter != null) { + verboseWriter.println(error); + } + } + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndex.java index 910dd93ff5..9545c0a279 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndex.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndex.java @@ -16,7 +16,11 @@ package com.google.gerrit.server.index.change; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.index.Index; +import com.google.gerrit.server.index.IndexDefinition; import com.google.gerrit.server.query.change.ChangeData; public interface ChangeIndex extends Index { + public interface Factory extends + IndexDefinition.IndexFactory { + } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndexDefintion.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndexDefintion.java new file mode 100644 index 0000000000..e2b429f54b --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndexDefintion.java @@ -0,0 +1,33 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.change; + +import com.google.gerrit.reviewdb.client.Change; +import com.google.gerrit.server.index.IndexDefinition; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.inject.Inject; + +public class ChangeIndexDefintion + extends IndexDefinition { + + @Inject + ChangeIndexDefintion( + ChangeIndexCollection indexCollection, + ChangeIndex.Factory indexFactory, + AllChangesIndexer allChangesIndexer) { + super(ChangeSchemaDefinitions.INSTANCE, indexCollection, indexFactory, + allChangesIndexer); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemas.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java similarity index 78% rename from gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemas.java rename to gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java index 60896d7e5a..b39a0b7a1f 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemas.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java @@ -1,4 +1,4 @@ -// Copyright (C) 2013 The Android Open Source Project +// Copyright (C) 2016 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,17 +14,13 @@ package com.google.gerrit.server.index.change; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.gerrit.server.index.SchemaUtil.schema; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.gerrit.server.index.Schema; -import com.google.gerrit.server.index.SchemaUtil; +import com.google.gerrit.server.index.SchemaDefinitions; import com.google.gerrit.server.query.change.ChangeData; -/** Secondary index schemas for changes. */ -public class ChangeSchemas { +public class ChangeSchemaDefinitions extends SchemaDefinitions { @Deprecated static final Schema V25 = schema( ChangeField.LEGACY_ID, @@ -102,16 +98,10 @@ public class ChangeSchemas { static final Schema V27 = schema(V26.getFields().values()); - public static final ImmutableMap> ALL = - SchemaUtil.schemasFromClass(ChangeSchemas.class, ChangeData.class); + public static final ChangeSchemaDefinitions INSTANCE = + new ChangeSchemaDefinitions(); - public static Schema get(int version) { - Schema schema = ALL.get(version); - checkArgument(schema != null, "Unrecognized schema version: %s", version); - return schema; - } - - public static Schema getLatest() { - return Iterables.getLast(ALL.values()); + private ChangeSchemaDefinitions() { + super("changes", ChangeData.class); } } diff --git a/gerrit-server/src/test/java/com/google/gerrit/testutil/InMemoryModule.java b/gerrit-server/src/test/java/com/google/gerrit/testutil/InMemoryModule.java index d4f098ec70..f6b6f95beb 100644 --- a/gerrit-server/src/test/java/com/google/gerrit/testutil/InMemoryModule.java +++ b/gerrit-server/src/test/java/com/google/gerrit/testutil/InMemoryModule.java @@ -14,7 +14,6 @@ package com.google.gerrit.testutil; -import static com.google.common.base.Preconditions.checkState; import static com.google.inject.Scopes.SINGLETON; import com.google.common.util.concurrent.MoreExecutors; @@ -49,7 +48,7 @@ import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.PerThreadRequestScope; import com.google.gerrit.server.git.SendEmailExecutor; import com.google.gerrit.server.index.IndexModule.IndexType; -import com.google.gerrit.server.index.change.ChangeSchemas; +import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier; import com.google.gerrit.server.notedb.NotesMigration; import com.google.gerrit.server.patch.DiffExecutor; @@ -74,10 +73,12 @@ import com.google.inject.servlet.RequestScoped; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.PersonIdent; -import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutorService; public class InMemoryModule extends FactoryModule { @@ -97,8 +98,6 @@ public class InMemoryModule extends FactoryModule { cfg.unset("cache", null, "directory"); cfg.setString("index", null, "type", "lucene"); cfg.setBoolean("index", "lucene", "testInmemory", true); - cfg.setInt("index", "lucene", "testVersion", - ChangeSchemas.getLatest().getVersion()); cfg.setInt("sendemail", null, "threadPoolSize", 0); cfg.setBoolean("receive", null, "enableSignedPush", false); cfg.setString("receive", null, "certNonceSeed", "sekret"); @@ -234,16 +233,20 @@ public class InMemoryModule extends FactoryModule { private Module luceneIndexModule() { try { + Map singleVersions = new HashMap<>(); int version = cfg.getInt("index", "lucene", "testVersion", -1); - checkState(ChangeSchemas.ALL.containsKey(version), - "invalid index.lucene.testVersion %s", version); + if (version > 0) { + singleVersions.put(ChangeSchemaDefinitions.INSTANCE.getName(), version); + } Class clazz = Class.forName("com.google.gerrit.lucene.LuceneIndexModule"); - Constructor c = clazz.getConstructor(Integer.class, int.class); - return (Module) c.newInstance(version, 0); + Method m = clazz.getMethod( + "singleVersionWithExplicitVersions", Map.class, int.class); + return (Module) m.invoke(null, singleVersions, 0); } catch (ClassNotFoundException | SecurityException | NoSuchMethodException - | IllegalArgumentException | InstantiationException - | IllegalAccessException | InvocationTargetException e) { + | IllegalArgumentException | IllegalAccessException + | InvocationTargetException e) { + e.printStackTrace(); ProvisionException pe = new ProvisionException(e.getMessage()); pe.initCause(e); throw pe; diff --git a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java index 20c3c2af03..bdf999aa6f 100644 --- a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java +++ b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java @@ -337,7 +337,7 @@ public class WebAppInitializer extends GuiceServletContextListener private Module createIndexModule() { switch (indexType) { case LUCENE: - return new LuceneIndexModule(); + return LuceneIndexModule.latestVersionWithOnlineUpgrade(); default: throw new IllegalStateException("unsupported index.type = " + indexType); }