From fc70ea396173a667b13272646997e222bb4d0756 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Mon, 21 Dec 2015 17:09:34 -0500 Subject: [PATCH] Remove support for pre-Lucene 5 indexes These hacks were added in 70aaf7ea to continue supporting old index versions stored on disk, which was necessary to upgrade v14 to v25 for 2.11->2.12. Now that stable-2.12 has been cut, we can remove support for this. Get rid of the core-and-backward-codecs target and just use core. A side effect of this change is that pack_war.py would now fail, since it expects each jar that goes in lib/ to have a unique name, and there is already a "core" target in MINA. Prefix all the lucene targets with "lucene-" to get it working. Change-Id: I146fc74ed649e0a44503f42bfa5822bf6fba5c03 --- gerrit-httpd/BUCK | 2 +- gerrit-lucene/BUCK | 8 +- .../gerrit/lucene/LuceneChangeIndex.java | 78 ++++--------------- gerrit-server/BUCK | 6 +- lib/asciidoctor/BUCK | 4 +- lib/lucene/BUCK | 39 ++-------- 6 files changed, 33 insertions(+), 104 deletions(-) diff --git a/gerrit-httpd/BUCK b/gerrit-httpd/BUCK index 3085054087..f4254dc6b2 100644 --- a/gerrit-httpd/BUCK +++ b/gerrit-httpd/BUCK @@ -36,7 +36,7 @@ java_library( '//lib/jgit:jgit', '//lib/jgit:jgit-servlet', '//lib/log:api', - '//lib/lucene:core-and-backward-codecs', + '//lib/lucene:lucene-core', ], provided_deps = ['//lib:servlet-api-3_1'], visibility = ['PUBLIC'], diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK index 8ba7479565..e9490d0f9b 100644 --- a/gerrit-lucene/BUCK +++ b/gerrit-lucene/BUCK @@ -11,7 +11,7 @@ java_library( '//gerrit-server:server', '//lib:gwtorm', '//lib:guava', - '//lib/lucene:core-and-backward-codecs', + '//lib/lucene:lucene-core', ], visibility = ['PUBLIC'], ) @@ -33,9 +33,9 @@ java_library( '//lib/guice:guice-assistedinject', '//lib/jgit:jgit', '//lib/log:api', - '//lib/lucene:analyzers-common', - '//lib/lucene:core-and-backward-codecs', - '//lib/lucene:misc', + '//lib/lucene:lucene-analyzers-common', + '//lib/lucene:lucene-core', + '//lib/lucene:lucene-misc', ], visibility = ['PUBLIC'], ) diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index 7778626eee..d3e662833c 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -15,7 +15,6 @@ package com.google.gerrit.lucene; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE; import static com.google.gerrit.server.index.ChangeField.LEGACY_ID2; @@ -74,8 +73,6 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -92,7 +89,6 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.BytesRef; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; @@ -137,6 +133,8 @@ public class LuceneChangeIndex implements ChangeIndex { ChangeField.REVIEWEDBY.getName(); private static final String UPDATED_SORT_FIELD = sortFieldName(ChangeField.UPDATED); + private static final String ID_SORT_FIELD = + sortFieldName(ChangeField.LEGACY_ID2); private static final Map CUSTOM_CHAR_MAPPING = ImmutableMap.of( "_", " ", ".", " "); @@ -206,18 +204,6 @@ public class LuceneChangeIndex implements ChangeIndex { private final QueryBuilder queryBuilder; private final SubIndex openIndex; private final SubIndex closedIndex; - private final String idSortField; - - /** - * Whether to use DocValues for range/sorted numeric fields. - *

- * Lucene 5 removed support for sorting based on normal numeric fields, so we - * use the newer API for more strongly typed numeric fields in newer schema - * versions. These fields also are not stored, so we need to store auxiliary - * stored-only field for them as well. - */ - // TODO(dborowitz): Delete when we delete support for pre-Lucene-5.0 schemas. - private final boolean useDocValuesForSorting; @AssistedInject LuceneChangeIndex( @@ -235,8 +221,6 @@ public class LuceneChangeIndex implements ChangeIndex { this.db = db; this.changeDataFactory = changeDataFactory; this.schema = schema; - this.useDocValuesForSorting = schema.getVersion() >= 15; - this.idSortField = sortFieldName(LEGACY_ID2); CustomMappingAnalyzer analyzer = new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET), @@ -251,7 +235,7 @@ public class LuceneChangeIndex implements ChangeIndex { GerritIndexWriterConfig closedConfig = new GerritIndexWriterConfig(cfg, "changes_closed"); - SearcherFactory searcherFactory = newSearcherFactory(); + SearcherFactory searcherFactory = new SearcherFactory(); if (cfg.getBoolean("index", "lucene", "testInmemory", false)) { openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig, searcherFactory); @@ -267,26 +251,6 @@ public class LuceneChangeIndex implements ChangeIndex { } } - private SearcherFactory newSearcherFactory() { - if (useDocValuesForSorting) { - return new SearcherFactory(); - } - final Map mapping = ImmutableMap.of( - // TODO(dborowitz): Remove; this is dead code anyway. - "_id", UninvertingReader.Type.INTEGER, - ChangeField.UPDATED.getName(), UninvertingReader.Type.LONG); - return new SearcherFactory() { - @Override - public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) - throws IOException { - checkState(reader instanceof DirectoryReader, - "expected DirectoryReader, found %s", reader.getClass().getName()); - return new IndexSearcher( - UninvertingReader.wrap((DirectoryReader) reader, mapping)); - } - }; - } - @Override public void close() { List> closeFutures = Lists.newArrayListWithCapacity(2); @@ -358,8 +322,7 @@ public class LuceneChangeIndex implements ChangeIndex { if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) { indexes.add(closedIndex); } - return new QuerySource(indexes, queryBuilder.toQuery(p), opts, - getSort()); + return new QuerySource(indexes, queryBuilder.toQuery(p), opts, getSort()); } @Override @@ -368,18 +331,9 @@ public class LuceneChangeIndex implements ChangeIndex { } private Sort getSort() { - if (useDocValuesForSorting) { - return new Sort( - new SortField(UPDATED_SORT_FIELD, SortField.Type.LONG, true), - new SortField(idSortField, SortField.Type.LONG, true)); - } else { - return new Sort( - new SortField( - ChangeField.UPDATED.getName(), SortField.Type.LONG, true), - new SortField( - // TODO(dborowitz): Remove; this is dead code anyway. - "_id", SortField.Type.INT, true)); - } + return new Sort( + new SortField(UPDATED_SORT_FIELD, SortField.Type.LONG, true), + new SortField(ID_SORT_FIELD, SortField.Type.LONG, true)); } public SubIndex getOpenChangesIndex() { @@ -599,15 +553,15 @@ public class LuceneChangeIndex implements ChangeIndex { FieldType type = values.getField().getType(); Store store = store(values.getField()); - if (useDocValuesForSorting) { - FieldDef f = values.getField(); - if (f == ChangeField.LEGACY_ID2) { - int v = (Integer) getOnlyElement(values.getValues()); - doc.add(new NumericDocValuesField(sortFieldName(f), v)); - } else if (f == ChangeField.UPDATED) { - long t = ((Timestamp) getOnlyElement(values.getValues())).getTime(); - doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t)); - } + FieldDef f = values.getField(); + + // Add separate DocValues fields for those fields needed for sorting. + if (f == ChangeField.LEGACY_ID2) { + int v = (Integer) getOnlyElement(values.getValues()); + doc.add(new NumericDocValuesField(sortFieldName(f), v)); + } else if (f == ChangeField.UPDATED) { + long t = ((Timestamp) getOnlyElement(values.getValues())).getTime(); + doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t)); } if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) { diff --git a/gerrit-server/BUCK b/gerrit-server/BUCK index b8fb4d241b..07847fc6e4 100644 --- a/gerrit-server/BUCK +++ b/gerrit-server/BUCK @@ -63,9 +63,9 @@ java_library( '//lib/log:api', '//lib/log:jsonevent-layout', '//lib/log:log4j', - '//lib/lucene:analyzers-common', - '//lib/lucene:core-and-backward-codecs', - '//lib/lucene:queryparser', + '//lib/lucene:lucene-analyzers-common', + '//lib/lucene:lucene-core', + '//lib/lucene:lucene-queryparser', '//lib/ow2:ow2-asm', '//lib/ow2:ow2-asm-tree', '//lib/ow2:ow2-asm-util', diff --git a/lib/asciidoctor/BUCK b/lib/asciidoctor/BUCK index ad13313d3e..f9b5b30087 100644 --- a/lib/asciidoctor/BUCK +++ b/lib/asciidoctor/BUCK @@ -35,8 +35,8 @@ java_library( '//gerrit-server:constants', '//lib:args4j', '//lib:guava', - '//lib/lucene:analyzers-common', - '//lib/lucene:core-and-backward-codecs', + '//lib/lucene:lucene-analyzers-common', + '//lib/lucene:lucene-core', ], visibility = ['//tools/eclipse:classpath'], ) diff --git a/lib/lucene/BUCK b/lib/lucene/BUCK index 7d0a16c56d..6bc48341f9 100644 --- a/lib/lucene/BUCK +++ b/lib/lucene/BUCK @@ -2,19 +2,8 @@ include_defs('//lib/maven.defs') VERSION = '5.3.1' -# core and backward-codecs both provide -# META-INF/services/org.apache.lucene.codecs.Codec, so they must be merged. -merge_maven_jars( - name = 'core-and-backward-codecs', - srcs = [ - ':backward-codecs_jar', - ':core_jar', - ], - visibility = ['PUBLIC'], -) - maven_jar( - name = 'core_jar', + name = 'lucene-core', id = 'org.apache.lucene:lucene-core:' + VERSION, sha1 = '36860653d7e09790ada96aeb1970b4ca396ac5d7', license = 'Apache2.0', @@ -22,15 +11,14 @@ maven_jar( 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt', ], - visibility = [], ) maven_jar( - name = 'analyzers-common', + name = 'lucene-analyzers-common', id = 'org.apache.lucene:lucene-analyzers-common:' + VERSION, sha1 = 'bd804dbc1b8f7941018926e940d20d1016b36c4c', license = 'Apache2.0', - deps = [':core-and-backward-codecs'], + deps = [':lucene-core'], exclude = [ 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt', @@ -38,24 +26,11 @@ maven_jar( ) maven_jar( - name = 'backward-codecs_jar', - id = 'org.apache.lucene:lucene-backward-codecs:' + VERSION, - sha1 = '380603f537317a78f9d9b7421bc2ac87586cb9a1', - license = 'Apache2.0', - deps = [':core_jar'], - exclude = [ - 'META-INF/LICENSE.txt', - 'META-INF/NOTICE.txt', - ], - visibility = [], -) - -maven_jar( - name = 'misc', + name = 'lucene-misc', id = 'org.apache.lucene:lucene-misc:' + VERSION, sha1 = '7891bbc18b372135c2a52b471075b0bdf5f110ec', license = 'Apache2.0', - deps = [':core-and-backward-codecs'], + deps = [':lucene-core'], exclude = [ 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt', @@ -63,11 +38,11 @@ maven_jar( ) maven_jar( - name = 'queryparser', + name = 'lucene-queryparser', id = 'org.apache.lucene:lucene-queryparser:' + VERSION, sha1 = 'bef0e2ac5b196dbab9d0b7c8cc8196b7ef5dd056', license = 'Apache2.0', - deps = [':core-and-backward-codecs'], + deps = [':lucene-core'], exclude = [ 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt',