diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK index 2b45d2bca9..a1467749da 100644 --- a/gerrit-lucene/BUCK +++ b/gerrit-lucene/BUCK @@ -34,7 +34,9 @@ java_library( '//lib/jgit:jgit', '//lib/log:api', '//lib/lucene:analyzers-common', + '//lib/lucene:backward-codecs', '//lib/lucene:core', + '//lib/lucene:misc', ], visibility = ['PUBLIC'], ) diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java index e0c13ae03a..27ded17170 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java @@ -14,7 +14,6 @@ package com.google.gerrit.lucene; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -42,13 +41,6 @@ class AutoCommitWriter extends IndexWriter { autoFlush(); } - @Override - public void addDocument(Iterable doc, - Analyzer analyzer) throws IOException { - super.addDocument(doc, analyzer); - autoFlush(); - } - @Override public void addDocuments( Iterable> docs) @@ -57,14 +49,6 @@ class AutoCommitWriter extends IndexWriter { autoFlush(); } - @Override - public void addDocuments( - Iterable> docs, - Analyzer analyzer) throws IOException { - super.addDocuments(docs, analyzer); - autoFlush(); - } - @Override public void updateDocuments(Term delTerm, Iterable> docs) @@ -73,14 +57,6 @@ class AutoCommitWriter extends IndexWriter { autoFlush(); } - @Override - public void updateDocuments(Term delTerm, - Iterable> docs, - Analyzer analyzer) throws IOException { - super.updateDocuments(delTerm, docs, analyzer); - autoFlush(); - } - @Override public void deleteDocuments(Term... term) throws IOException { super.deleteDocuments(term); @@ -110,13 +86,6 @@ class AutoCommitWriter extends IndexWriter { autoFlush(); } - @Override - public void updateDocument(Term term, Iterable doc, - Analyzer analyzer) throws IOException { - super.updateDocument(term, doc, analyzer); - autoFlush(); - } - @Override public void deleteAll() throws IOException { super.deleteAll(); diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index 0084536a47..15b8fc3c99 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -14,10 +14,12 @@ package com.google.gerrit.lucene; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE; import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES; import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES; + import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; @@ -39,7 +41,6 @@ import com.google.gerrit.server.index.ChangeField; import com.google.gerrit.server.index.ChangeField.ChangeProtoField; import com.google.gerrit.server.index.ChangeField.PatchSetApprovalProtoField; import com.google.gerrit.server.index.ChangeIndex; -import com.google.gerrit.server.index.ChangeSchemas; import com.google.gerrit.server.index.FieldDef; import com.google.gerrit.server.index.FieldDef.FillArgs; import com.google.gerrit.server.index.FieldType; @@ -64,9 +65,12 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.IntField; import org.apache.lucene.document.LongField; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -76,13 +80,14 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Version; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -120,54 +125,19 @@ public class LuceneChangeIndex implements ChangeIndex { private static final String CHANGE_FIELD = ChangeField.CHANGE.getName(); private static final String DELETED_FIELD = ChangeField.DELETED.getName(); private static final String ID_FIELD = ChangeField.LEGACY_ID.getName(); + private static final String ID_SORT_FIELD = + sortFieldName(ChangeField.LEGACY_ID); private static final String MERGEABLE_FIELD = ChangeField.MERGEABLE.getName(); + private static final String UPDATED_SORT_FIELD = + sortFieldName(ChangeField.UPDATED); + private static final ImmutableSet FIELDS = ImmutableSet.of( ADDED_FIELD, APPROVAL_FIELD, CHANGE_FIELD, DELETED_FIELD, ID_FIELD, MERGEABLE_FIELD); + private static final Map CUSTOM_CHAR_MAPPING = ImmutableMap.of( "_", " ", ".", " "); - private static final Map, Version> LUCENE_VERSIONS; - static { - ImmutableMap.Builder, Version> versions = - ImmutableMap.builder(); - @SuppressWarnings("deprecation") - Version lucene43 = Version.LUCENE_43; - @SuppressWarnings("deprecation") - Version lucene44 = Version.LUCENE_44; - @SuppressWarnings("deprecation") - Version lucene46 = Version.LUCENE_46; - @SuppressWarnings("deprecation") - Version lucene47 = Version.LUCENE_47; - @SuppressWarnings("deprecation") - Version lucene48 = Version.LUCENE_48; - @SuppressWarnings("deprecation") - Version lucene410 = Version.LUCENE_4_10_0; - // We are using 4.10.2 but there is no difference in the index - // format since 4.10.1, so we reuse the version here. - @SuppressWarnings("deprecation") - Version lucene4101 = Version.LUCENE_4_10_1; - for (Map.Entry> e - : ChangeSchemas.ALL.entrySet()) { - if (e.getKey() <= 3) { - versions.put(e.getValue(), lucene43); - } else if (e.getKey() <= 5) { - versions.put(e.getValue(), lucene44); - } else if (e.getKey() <= 8) { - versions.put(e.getValue(), lucene46); - } else if (e.getKey() <= 10) { - versions.put(e.getValue(), lucene47); - } else if (e.getKey() <= 11) { - versions.put(e.getValue(), lucene48); - } else if (e.getKey() <= 13) { - versions.put(e.getValue(), lucene410); - } else { - versions.put(e.getValue(), lucene4101); - } - } - LUCENE_VERSIONS = versions.build(); - } - public static void setReady(SitePaths sitePaths, int version, boolean ready) throws IOException { try { @@ -180,6 +150,10 @@ public class LuceneChangeIndex implements ChangeIndex { } } + private static String sortFieldName(FieldDef f) { + return f.getName() + "_SORT"; + } + static interface Factory { LuceneChangeIndex create(Schema schema, String base); } @@ -188,12 +162,13 @@ public class LuceneChangeIndex implements ChangeIndex { private final IndexWriterConfig luceneConfig; private long commitWithinMs; - private GerritIndexWriterConfig(Version version, Config cfg, String name) { + private GerritIndexWriterConfig(Config cfg, String name) { CustomMappingAnalyzer analyzer = new CustomMappingAnalyzer(new StandardAnalyzer( CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING); - luceneConfig = new IndexWriterConfig(version, analyzer); - luceneConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); + luceneConfig = new IndexWriterConfig(analyzer) + .setOpenMode(OpenMode.CREATE_OR_APPEND) + .setCommitOnClose(true); double m = 1 << 20; luceneConfig.setRAMBufferSizeMB(cfg.getLong( "index", name, "ramBufferSize", @@ -229,6 +204,17 @@ public class LuceneChangeIndex implements ChangeIndex { private final SubIndex openIndex; private final SubIndex closedIndex; + /** + * Whether to use DocValues for range/sorted numeric fields. + *

+ * Lucene 5 removed support for sorting based on normal numeric fields, so we + * use the newer API for more strongly typed numeric fields in newer schema + * versions. These fields also are not stored, so we need to store auxiliary + * stored-only field for them as well. + */ + // TODO(dborowitz): Delete when we delete support for pre-Lucene-5.0 schemas. + private final boolean useDocValuesForSorting; + @AssistedInject LuceneChangeIndex( @GerritServerConfig Config cfg, @@ -245,10 +231,8 @@ public class LuceneChangeIndex implements ChangeIndex { this.db = db; this.changeDataFactory = changeDataFactory; this.schema = schema; + this.useDocValuesForSorting = schema.getVersion() >= 15; - Version luceneVersion = checkNotNull( - LUCENE_VERSIONS.get(schema), - "unknown Lucene version for index schema: %s", schema); CustomMappingAnalyzer analyzer = new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING); @@ -258,21 +242,44 @@ public class LuceneChangeIndex implements ChangeIndex { BooleanQuery.getMaxClauseCount())); GerritIndexWriterConfig openConfig = - new GerritIndexWriterConfig(luceneVersion, cfg, "changes_open"); + new GerritIndexWriterConfig(cfg, "changes_open"); GerritIndexWriterConfig closedConfig = - new GerritIndexWriterConfig(luceneVersion, cfg, "changes_closed"); + new GerritIndexWriterConfig(cfg, "changes_closed"); + SearcherFactory searcherFactory = newSearcherFactory(); if (cfg.getBoolean("index", "lucene", "testInmemory", false)) { - openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig); - closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig); + openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig, + searcherFactory); + closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig, + searcherFactory); } else { Path dir = base != null ? Paths.get(base) : LuceneVersionManager.getDir(sitePaths, schema); - openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig); - closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig); + openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig, + searcherFactory); + closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig, + searcherFactory); } } + private SearcherFactory newSearcherFactory() { + if (useDocValuesForSorting) { + return new SearcherFactory(); + } + final Map mapping = ImmutableMap.of( + ChangeField.LEGACY_ID.getName(), UninvertingReader.Type.INTEGER, + ChangeField.UPDATED.getName(), UninvertingReader.Type.LONG); + return new SearcherFactory() { + @Override + public IndexSearcher newSearcher(IndexReader reader) { + checkState(reader instanceof DirectoryReader, + "expected DirectoryReader, found %s", reader.getClass().getName()); + return new IndexSearcher( + UninvertingReader.wrap((DirectoryReader) reader, mapping)); + } + }; + } + @Override public void close() { List> closeFutures = Lists.newArrayListWithCapacity(2); @@ -355,12 +362,18 @@ public class LuceneChangeIndex implements ChangeIndex { setReady(sitePaths, schema.getVersion(), ready); } - private static Sort getSort() { - return new Sort( - new SortField( - ChangeField.UPDATED.getName(), SortField.Type.LONG, true), - new SortField( - ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true)); + private Sort getSort() { + if (useDocValuesForSorting) { + return new Sort( + new SortField(UPDATED_SORT_FIELD, SortField.Type.LONG, true), + new SortField(ID_SORT_FIELD, SortField.Type.LONG, true)); + } else { + return new Sort( + new SortField( + ChangeField.UPDATED.getName(), SortField.Type.LONG, true), + new SortField( + ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true)); + } } private class QuerySource implements ChangeDataSource { @@ -506,6 +519,16 @@ public class LuceneChangeIndex implements ChangeIndex { FieldType type = values.getField().getType(); Store store = store(values.getField()); + if (useDocValuesForSorting) { + if (values.getField() == ChangeField.LEGACY_ID) { + int v = (Integer) getOnlyElement(values.getValues()); + doc.add(new NumericDocValuesField(ID_SORT_FIELD, v)); + } else if (values.getField() == ChangeField.UPDATED) { + long t = ((Timestamp) getOnlyElement(values.getValues())).getTime(); + doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t)); + } + } + if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) { for (Object value : values.getValues()) { doc.add(new IntField(name, (Integer) value, store)); diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java index f28bf05327..5778008f8c 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java @@ -28,9 +28,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TrackingIndexWriter; import org.apache.lucene.search.ControlledRealTimeReopenThread; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.ReferenceManager.RefreshListener; import org.apache.lucene.search.SearcherFactory; -import org.apache.lucene.search.SearcherManager; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -52,17 +52,19 @@ class SubIndex { private final Directory dir; private final TrackingIndexWriter writer; - private final SearcherManager searcherManager; + private final ReferenceManager searcherManager; private final ControlledRealTimeReopenThread reopenThread; private final Set notDoneNrtFutures; - SubIndex(Path path, GerritIndexWriterConfig writerConfig) throws IOException { - this(FSDirectory.open(path.toFile()), path.getFileName().toString(), - writerConfig); + SubIndex(Path path, GerritIndexWriterConfig writerConfig, + SearcherFactory searcherFactory) throws IOException { + this(FSDirectory.open(path), path.getFileName().toString(), writerConfig, + searcherFactory); } SubIndex(Directory dir, final String dirName, - GerritIndexWriterConfig writerConfig) throws IOException { + GerritIndexWriterConfig writerConfig, + SearcherFactory searcherFactory) throws IOException { this.dir = dir; IndexWriter delegateWriter; long commitPeriod = writerConfig.getCommitWithinMs(); @@ -104,8 +106,8 @@ class SubIndex { }, commitPeriod, commitPeriod, MILLISECONDS); } writer = new TrackingIndexWriter(delegateWriter); - searcherManager = new SearcherManager( - writer.getIndexWriter(), true, new SearcherFactory()); + searcherManager = new WrappableSearcherManager( + writer.getIndexWriter(), true, searcherFactory); notDoneNrtFutures = Sets.newConcurrentHashSet(); @@ -125,6 +127,8 @@ class SubIndex { // searching generation being up to date when calling // reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's // internal listener needs to be called first. + // TODO(dborowitz): This may have been fixed by + // http://issues.apache.org/jira/browse/LUCENE-5461 searcherManager.addListener(new RefreshListener() { @Override public void beforeRefresh() throws IOException { @@ -158,12 +162,9 @@ class SubIndex { } try { - writer.getIndexWriter().commit(); - try { - writer.getIndexWriter().close(); - } catch (AlreadyClosedException e) { - // Ignore. - } + writer.getIndexWriter().close(); + } catch (AlreadyClosedException e) { + // Ignore. } catch (IOException e) { log.warn("error closing Lucene writer", e); } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java new file mode 100644 index 0000000000..fe45f1ddc9 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java @@ -0,0 +1,217 @@ +package com.google.gerrit.lucene; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.store.Directory; + +/** + * Utility class to safely share {@link IndexSearcher} instances across multiple + * threads, while periodically reopening. This class ensures each searcher is + * closed only once all threads have finished using it. + * + *

+ * Use {@link #acquire} to obtain the current searcher, and {@link #release} to + * release it, like this: + * + * 

+ * IndexSearcher s = manager.acquire();
+ * try {
+ *   // Do searching, doc retrieval, etc. with s
+ * } finally {
+ *   manager.release(s);
+ * }
+ * // Do not use s after this!
+ * s = null;
+ *
+ * + *

+ * In addition you should periodically call {@link #maybeRefresh}. While it's + * possible to call this just before running each query, this is discouraged + * since it penalizes the unlucky queries that need to refresh. It's better to use + * a separate background thread, that periodically calls {@link #maybeRefresh}. Finally, + * be sure to call {@link #close} once you are done. + * + * @see SearcherFactory + * + * @lucene.experimental + */ +// This file was copied from: +// https://github.com/apache/lucene-solr/blob/lucene_solr_5_0/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java +// The only change (other than class name and import fixes) +// is to skip the check in getSearcher that searcherFactory.newSearcher wraps +// the provided searcher exactly. +final class WrappableSearcherManager extends ReferenceManager { + + private final SearcherFactory searcherFactory; + + /** + * Creates and returns a new SearcherManager from the given + * {@link IndexWriter}. + * + * @param writer + * the IndexWriter to open the IndexReader from. + * @param applyAllDeletes + * If true, all buffered deletes will be applied (made + * visible) in the {@link IndexSearcher} / {@link DirectoryReader}. + * If false, the deletes may or may not be applied, but + * remain buffered (in IndexWriter) so that they will be applied in + * the future. Applying deletes can be costly, so if your app can + * tolerate deleted documents being returned you might gain some + * performance by passing false. See + * {@link DirectoryReader#openIfChanged(DirectoryReader, IndexWriter, boolean)}. + * @param searcherFactory + * An optional {@link SearcherFactory}. Pass null if you + * don't require the searcher to be warmed before going live or other + * custom behavior. + * + * @throws IOException if there is a low-level I/O error + */ + public WrappableSearcherManager(IndexWriter writer, boolean applyAllDeletes, SearcherFactory searcherFactory) throws IOException { + if (searcherFactory == null) { + searcherFactory = new SearcherFactory(); + } + this.searcherFactory = searcherFactory; + current = getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes)); + } + + /** + * Creates and returns a new SearcherManager from the given {@link Directory}. + * @param dir the directory to open the DirectoryReader on. + * @param searcherFactory An optional {@link SearcherFactory}. Pass + * null if you don't require the searcher to be warmed + * before going live or other custom behavior. + * + * @throws IOException if there is a low-level I/O error + */ + public WrappableSearcherManager(Directory dir, SearcherFactory searcherFactory) throws IOException { + if (searcherFactory == null) { + searcherFactory = new SearcherFactory(); + } + this.searcherFactory = searcherFactory; + current = getSearcher(searcherFactory, DirectoryReader.open(dir)); + } + + /** + * Creates and returns a new SearcherManager from an existing {@link DirectoryReader}. Note that + * this steals the incoming reference. + * + * @param reader the DirectoryReader. + * @param searcherFactory An optional {@link SearcherFactory}. Pass + * null if you don't require the searcher to be warmed + * before going live or other custom behavior. + * + * @throws IOException if there is a low-level I/O error + */ + public WrappableSearcherManager(DirectoryReader reader, SearcherFactory searcherFactory) throws IOException { + if (searcherFactory == null) { + searcherFactory = new SearcherFactory(); + } + this.searcherFactory = searcherFactory; + this.current = getSearcher(searcherFactory, reader); + } + + @Override + protected void decRef(IndexSearcher reference) throws IOException { + reference.getIndexReader().decRef(); + } + + @Override + protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException { + final IndexReader r = referenceToRefresh.getIndexReader(); + assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r; + final IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader) r); + if (newReader == null) { + return null; + } else { + return getSearcher(searcherFactory, newReader); + } + } + + @Override + protected boolean tryIncRef(IndexSearcher reference) { + return reference.getIndexReader().tryIncRef(); + } + + @Override + protected int getRefCount(IndexSearcher reference) { + return reference.getIndexReader().getRefCount(); + } + + /** + * Returns true if no changes have occured since this searcher + * ie. reader was opened, otherwise false. + * @see DirectoryReader#isCurrent() + */ + public boolean isSearcherCurrent() throws IOException { + final IndexSearcher searcher = acquire(); + try { + final IndexReader r = searcher.getIndexReader(); + assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r; + return ((DirectoryReader) r).isCurrent(); + } finally { + release(searcher); + } + } + + /** Expert: creates a searcher from the provided {@link + * IndexReader} using the provided {@link + * SearcherFactory}. NOTE: this decRefs incoming reader + * on throwing an exception. */ + @SuppressWarnings("resource") + public static IndexSearcher getSearcher(SearcherFactory searcherFactory, IndexReader reader) throws IOException { + boolean success = false; + final IndexSearcher searcher; + try { + searcher = searcherFactory.newSearcher(reader); + // Modification for Gerrit: Allow searcherFactory to transitively wrap the + // provided reader. + IndexReader unwrapped = searcher.getIndexReader(); + while (true) { + if (unwrapped == reader) { + break; + } else if (unwrapped instanceof FilterDirectoryReader) { + unwrapped = ((FilterDirectoryReader) unwrapped).getDelegate(); + } else if (unwrapped instanceof FilterLeafReader) { + unwrapped = ((FilterLeafReader) unwrapped).getDelegate(); + } else { + break; + } + } + + if (unwrapped != reader) { + throw new IllegalStateException("SearcherFactory must wrap the provided reader (got " + searcher.getIndexReader() + " but expected " + reader + ")"); + } + success = true; + } finally { + if (!success) { + reader.decRef(); + } + } + return searcher; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java index 821343f457..00407625bc 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java @@ -55,6 +55,10 @@ import java.util.Set; * {@link ChangeQueryBuilder} for querying that field, and a method on * {@link ChangeData} used for populating the corresponding document fields in * the secondary index. + *

+ * Field names are all lowercase alphanumeric plus underscore; index + * implementations may create unambiguous derived field names containing other + * characters. */ public class ChangeField { /** Legacy change ID. */ diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java index 557faebefa..cf3fd09369 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java @@ -14,6 +14,9 @@ package com.google.gerrit.server.index; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.CharMatcher; import com.google.common.base.Preconditions; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.TrackingFooters; @@ -78,11 +81,17 @@ public abstract class FieldDef { private final boolean stored; private FieldDef(String name, FieldType type, boolean stored) { - this.name = name; + this.name = checkName(name); this.type = type; this.stored = stored; } + private static String checkName(String name) { + CharMatcher m = CharMatcher.anyOf("abcdefghijklmnopqrstuvwxyz0123456789_"); + checkArgument(m.matchesAllOf(name), "illegal field name: %s", name); + return name; + } + /** @return name of the field. */ public final String getName() { return name; diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java new file mode 100644 index 0000000000..42f50723a7 --- /dev/null +++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java @@ -0,0 +1,49 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.change; + +import com.google.gerrit.testutil.InMemoryModule; +import com.google.inject.Guice; +import com.google.inject.Injector; + +import org.eclipse.jgit.lib.Config; +import org.junit.Ignore; +import org.junit.Test; + +public class LuceneQueryChangesV14Test extends LuceneQueryChangesTest { + + @Override + protected Injector createInjector() { + Config luceneConfig = new Config(config); + InMemoryModule.setDefaults(luceneConfig); + // Latest version with a Lucene 4 index. + luceneConfig.setInt("index", "lucene", "testVersion", 14); + return Guice.createInjector(new InMemoryModule(luceneConfig)); + } + + @Override + @Ignore + @Test + public void byCommentBy() { + // Ignore. + } + + @Override + @Ignore + @Test + public void byFrom() { + // Ignore. + } +} diff --git a/lib/asciidoctor/java/DocIndexer.java b/lib/asciidoctor/java/DocIndexer.java index 7eb70c1243..f06c6629d6 100644 --- a/lib/asciidoctor/java/DocIndexer.java +++ b/lib/asciidoctor/java/DocIndexer.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Version; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; @@ -51,8 +50,6 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; public class DocIndexer { - @SuppressWarnings("deprecation") - private static final Version LUCENE_VERSION = Version.LUCENE_4_10_1; private static final Pattern SECTION_HEADER = Pattern.compile("^=+ (.*)"); @Option(name = "-o", usage = "output JAR file") @@ -99,9 +96,9 @@ public class DocIndexer { UnsupportedEncodingException, FileNotFoundException { RAMDirectory directory = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig( - LUCENE_VERSION, new StandardAnalyzer(CharArraySet.EMPTY_SET)); config.setOpenMode(OpenMode.CREATE); + config.setCommitOnClose(true); IndexWriter iwriter = new IndexWriter(directory, config); for (String inputFile : inputFiles) { File file = new File(inputFile); diff --git a/lib/lucene/BUCK b/lib/lucene/BUCK index 9026f79605..275f0bb0f6 100644 --- a/lib/lucene/BUCK +++ b/lib/lucene/BUCK @@ -1,11 +1,11 @@ include_defs('//lib/maven.defs') -VERSION = '4.10.2' +VERSION = '5.0.0' maven_jar( name = 'core', id = 'org.apache.lucene:lucene-core:' + VERSION, - sha1 = 'c01e3d675d277e0a93e7890d03cc3246b2cdecaa', + sha1 = '4395e5ea987af804c4a9b96131e2ee75db061fdf', license = 'Apache2.0', exclude = [ 'META-INF/LICENSE.txt', @@ -16,8 +16,33 @@ maven_jar( maven_jar( name = 'analyzers-common', id = 'org.apache.lucene:lucene-analyzers-common:' + VERSION, - sha1 = 'f977f8c443e8f4e9d1fd7fdfda80a6cf60b3e7c2', + sha1 = '6159cbc5c9631ef75e1f0e97b358ecdd8f1447a9', license = 'Apache2.0', + deps = [':core'], + exclude = [ + 'META-INF/LICENSE.txt', + 'META-INF/NOTICE.txt', + ], +) + +maven_jar( + name = 'backward-codecs', + id = 'org.apache.lucene:lucene-backward-codecs:' + VERSION, + sha1 = '5cd11fc1be436ff96b63f0f76f299a9d25543b0b', + license = 'Apache2.0', + deps = [':core'], + exclude = [ + 'META-INF/LICENSE.txt', + 'META-INF/NOTICE.txt', + ], +) + +maven_jar( + name = 'misc', + id = 'org.apache.lucene:lucene-misc:' + VERSION, + sha1 = '06bd7cb030e598da81a8228f5c58630e5ce7b84a', + license = 'Apache2.0', + deps = [':core'], exclude = [ 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt', @@ -27,6 +52,11 @@ maven_jar( maven_jar( name = 'query-parser', id = 'org.apache.lucene:lucene-queryparser:' + VERSION, - sha1 = 'd70f54e1060d553ba7aeb4d49a71fd0c068499e8', + sha1 = 'f459326c0b58bb837612bfeb37f6015c1a8962db', license = 'Apache2.0', + deps = [':core'], + exclude = [ + 'META-INF/LICENSE.txt', + 'META-INF/NOTICE.txt', + ], )