diff --git a/gerrit-acceptance-tests/BUCK b/gerrit-acceptance-tests/BUCK index 6b4c2bf044..f3a27ff581 100644 --- a/gerrit-acceptance-tests/BUCK +++ b/gerrit-acceptance-tests/BUCK @@ -24,13 +24,13 @@ java_test( '//lib:junit', '//lib:servlet-api-3_0', + '//lib/commons:httpclient', + '//lib/commons:httpcore', '//lib/log:impl_log4j', '//lib/log:log4j', '//lib/guice:guice', '//lib/jgit:jgit', '//lib/jgit:junit', - '//lib/openid:httpclient', - '//lib/openid:httpcore', ], source_under_test = TEST, labels = ['slow'], diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK index 927b66a161..67dd218ecd 100644 --- a/gerrit-lucene/BUCK +++ b/gerrit-lucene/BUCK @@ -1,18 +1,36 @@ +QUERY_BUILDER = [ + 'src/main/java/com/google/gerrit/lucene/QueryBuilder.java', +] + +java_library( + name = 'query_builder', + srcs = QUERY_BUILDER, + deps = [ + '//gerrit-antlr:query_exception', + '//gerrit-reviewdb:server', + '//gerrit-server:server', + '//lib:gwtorm', + '//lib/lucene:core', + ], + visibility = ['PUBLIC'], +) + java_library( name = 'lucene', - srcs = glob(['src/main/java/**/*.java']), + srcs = glob(['src/main/java/**/*.java'], excludes = QUERY_BUILDER), deps = [ + ':query_builder', '//gerrit-antlr:query_exception', '//gerrit-extension-api:api', '//gerrit-reviewdb:server', '//gerrit-server:server', '//lib:guava', '//lib:gwtorm', - '//lib:lucene-analyzers-common', - '//lib:lucene-core', '//lib/guice:guice', '//lib/jgit:jgit', '//lib/log:api', + '//lib/lucene:analyzers-common', + '//lib/lucene:core', ], visibility = ['PUBLIC'], ) diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index 27581be29b..d352b37962 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -14,11 +14,10 @@ package com.google.gerrit.lucene; +import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS; +import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig; import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES; import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES; -import static org.apache.lucene.search.BooleanClause.Occur.MUST; -import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT; -import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; import com.google.common.base.Function; import com.google.common.collect.ImmutableSet; @@ -35,18 +34,11 @@ import com.google.gerrit.server.index.ChangeIndex; import com.google.gerrit.server.index.FieldDef; import com.google.gerrit.server.index.FieldDef.FillArgs; import com.google.gerrit.server.index.FieldType; -import com.google.gerrit.server.index.IndexPredicate; -import com.google.gerrit.server.index.RegexPredicate; -import com.google.gerrit.server.index.TimestampRangePredicate; -import com.google.gerrit.server.query.AndPredicate; -import com.google.gerrit.server.query.NotPredicate; -import com.google.gerrit.server.query.OrPredicate; import com.google.gerrit.server.query.Predicate; import com.google.gerrit.server.query.QueryParseException; import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.ChangeDataSource; import com.google.gerrit.server.query.change.IndexRewriteImpl; -import com.google.gerrit.server.query.change.SortKeyPredicate; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.ResultSet; @@ -62,24 +54,18 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.NumericRangeQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; +import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +75,7 @@ import java.sql.Timestamp; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -122,6 +109,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { return writerConfig; } + private final SitePaths sitePaths; private final FillArgs fillArgs; private final ExecutorService executor; private final boolean readOnly; @@ -131,6 +119,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { LuceneChangeIndex(Config cfg, SitePaths sitePaths, ListeningScheduledExecutorService executor, FillArgs fillArgs, boolean readOnly) throws IOException { + this.sitePaths = sitePaths; this.fillArgs = fillArgs; this.executor = executor; this.readOnly = readOnly; @@ -167,7 +156,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { @SuppressWarnings("unchecked") @Override public ListenableFuture insert(ChangeData cd) throws IOException { - Term id = idTerm(cd); + Term id = QueryBuilder.idTerm(cd); Document doc = toDocument(cd); if (readOnly) { return Futures.immediateFuture(null); @@ -187,7 +176,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { @SuppressWarnings("unchecked") @Override public ListenableFuture replace(ChangeData cd) throws IOException { - Term id = idTerm(cd); + Term id = QueryBuilder.idTerm(cd); Document doc = toDocument(cd); if (readOnly) { return Futures.immediateFuture(null); @@ -206,7 +195,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { @SuppressWarnings("unchecked") @Override public ListenableFuture delete(ChangeData cd) throws IOException { - Term id = idTerm(cd); + Term id = QueryBuilder.idTerm(cd); if (readOnly) { return Futures.immediateFuture(null); } @@ -226,6 +215,11 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { }); } + @Override + public void deleteAll() throws IOException { + openIndex.deleteAll(); + } + @Override public ChangeDataSource getSource(Predicate p) throws QueryParseException { @@ -237,138 +231,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) { indexes.add(closedIndex); } - return new QuerySource(indexes, toQuery(p)); - } - - private Term idTerm(ChangeData cd) { - return intTerm(ID_FIELD, cd.getId().get()); - } - - private Query toQuery(Predicate p) throws QueryParseException { - if (p.getClass() == AndPredicate.class) { - return booleanQuery(p, MUST); - } else if (p.getClass() == OrPredicate.class) { - return booleanQuery(p, SHOULD); - } else if (p.getClass() == NotPredicate.class) { - if (p.getChild(0) instanceof TimestampRangePredicate) { - return notTimestampQuery( - (TimestampRangePredicate) p.getChild(0)); - } - return booleanQuery(p, MUST_NOT); - } else if (p instanceof IndexPredicate) { - return fieldQuery((IndexPredicate) p); - } else { - throw new QueryParseException("Cannot convert to index predicate: " + p); - } - } - - private Query booleanQuery(Predicate p, BooleanClause.Occur o) - throws QueryParseException { - BooleanQuery q = new BooleanQuery(); - for (int i = 0; i < p.getChildCount(); i++) { - q.add(toQuery(p.getChild(i)), o); - } - return q; - } - - private Query fieldQuery(IndexPredicate p) - throws QueryParseException { - if (p.getType() == FieldType.INTEGER) { - return intQuery(p); - } else if (p.getType() == FieldType.TIMESTAMP) { - return timestampQuery(p); - } else if (p.getType() == FieldType.EXACT) { - return exactQuery(p); - } else if (p.getType() == FieldType.PREFIX) { - return prefixQuery(p); - } else if (p.getType() == FieldType.FULL_TEXT) { - return fullTextQuery(p); - } else if (p instanceof SortKeyPredicate) { - return sortKeyQuery((SortKeyPredicate) p); - } else { - throw badFieldType(p.getType()); - } - } - - private Term intTerm(String name, int value) { - BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT); - NumericUtils.intToPrefixCodedBytes(value, 0, bytes); - return new Term(name, bytes); - } - - private Query intQuery(IndexPredicate p) - throws QueryParseException { - int value; - try { - // Can't use IntPredicate because it and IndexPredicate are different - // subclasses of OperatorPredicate. - value = Integer.valueOf(p.getValue()); - } catch (IllegalArgumentException e) { - throw new QueryParseException("not an integer: " + p.getValue()); - } - return new TermQuery(intTerm(p.getField().getName(), value)); - } - - private static Query sortKeyQuery(SortKeyPredicate p) { - return NumericRangeQuery.newLongRange( - p.getField().getName(), - p.getMinValue(), - p.getMaxValue(), - true, true); - } - - private static Query timestampQuery(IndexPredicate p) - throws QueryParseException { - if (p instanceof TimestampRangePredicate) { - TimestampRangePredicate r = - (TimestampRangePredicate) p; - return NumericRangeQuery.newIntRange( - r.getField().getName(), - toIndexTime(r.getMinTimestamp()), - toIndexTime(r.getMaxTimestamp()), - true, true); - } - throw new QueryParseException("not a timestamp: " + p); - } - - private static Query notTimestampQuery(TimestampRangePredicate r) - throws QueryParseException { - if (r.getMinTimestamp().getTime() == 0) { - return NumericRangeQuery.newIntRange( - r.getField().getName(), - toIndexTime(r.getMaxTimestamp()), - null, - true, true); - } - throw new QueryParseException("cannot negate: " + r); - } - - private Query exactQuery(IndexPredicate p) { - if (p instanceof RegexPredicate) { - return regexQuery(p); - } else { - return new TermQuery(new Term(p.getField().getName(), p.getValue())); - } - } - - private Query regexQuery(IndexPredicate p) { - String re = p.getValue(); - if (re.startsWith("^")) { - re = re.substring(1); - } - if (re.endsWith("$") && !re.endsWith("\\$")) { - re = re.substring(0, re.length() - 1); - } - - return new RegexpQuery(new Term(p.getField().getName(), re)); - } - - private Query prefixQuery(IndexPredicate p) { - return new PrefixQuery(new Term(p.getField().getName(), p.getValue())); - } - - private Query fullTextQuery(IndexPredicate p) { - return new FuzzyQuery(new Term(p.getField().getName(), p.getValue())); + return new QuerySource(indexes, QueryBuilder.toQuery(p)); } private static class QuerySource implements ChangeDataSource { @@ -485,7 +348,8 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { } } else if (f.getType() == FieldType.TIMESTAMP) { for (Object v : values) { - doc.add(new IntField(name, toIndexTime((Timestamp) v), store)); + int t = QueryBuilder.toIndexTime((Timestamp) v); + doc.add(new IntField(name, t, store)); } } else if (f.getType() == FieldType.EXACT || f.getType() == FieldType.PREFIX) { @@ -497,19 +361,24 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { doc.add(new TextField(name, (String) value, store)); } } else { - throw badFieldType(f.getType()); + throw QueryBuilder.badFieldType(f.getType()); } } - private static int toIndexTime(Timestamp ts) { - return (int) (ts.getTime() / 60000); - } - private static Field.Store store(FieldDef f) { return f.isStored() ? Field.Store.YES : Field.Store.NO; } - private static IllegalArgumentException badFieldType(FieldType t) { - return new IllegalArgumentException("unknown index field type " + t); + @Override + public void finishIndex() throws IOException, + ConfigInvalidException { + FileBasedConfig cfg = + new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect()); + + for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { + cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue()); + } + cfg.setEnum("lucene", null, "version", LUCENE_VERSION); + cfg.save(); } } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java new file mode 100644 index 0000000000..4f57a5a9d4 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java @@ -0,0 +1,192 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.lucene; + +import static org.apache.lucene.search.BooleanClause.Occur.MUST; +import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT; +import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; + +import com.google.gerrit.server.index.ChangeField; +import com.google.gerrit.server.index.FieldType; +import com.google.gerrit.server.index.IndexPredicate; +import com.google.gerrit.server.index.RegexPredicate; +import com.google.gerrit.server.index.TimestampRangePredicate; +import com.google.gerrit.server.query.AndPredicate; +import com.google.gerrit.server.query.NotPredicate; +import com.google.gerrit.server.query.OrPredicate; +import com.google.gerrit.server.query.Predicate; +import com.google.gerrit.server.query.QueryParseException; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.gerrit.server.query.change.SortKeyPredicate; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +import java.sql.Timestamp; + +public class QueryBuilder { + private static final String ID_FIELD = ChangeField.LEGACY_ID.getName(); + + public static Term idTerm(ChangeData cd) { + return intTerm(ID_FIELD, cd.getId().get()); + } + + public static Query toQuery(Predicate p) + throws QueryParseException { + if (p.getClass() == AndPredicate.class) { + return booleanQuery(p, MUST); + } else if (p.getClass() == OrPredicate.class) { + return booleanQuery(p, SHOULD); + } else if (p.getClass() == NotPredicate.class) { + if (p.getChild(0) instanceof TimestampRangePredicate) { + return notTimestampQuery( + (TimestampRangePredicate) p.getChild(0)); + } + return booleanQuery(p, MUST_NOT); + } else if (p instanceof IndexPredicate) { + return fieldQuery((IndexPredicate) p); + } else { + throw new QueryParseException("Cannot convert to index predicate: " + p); + } + } + + private static Query booleanQuery(Predicate p, BooleanClause.Occur o) + throws QueryParseException { + BooleanQuery q = new BooleanQuery(); + for (int i = 0; i < p.getChildCount(); i++) { + q.add(toQuery(p.getChild(i)), o); + } + return q; + } + + private static Query fieldQuery(IndexPredicate p) + throws QueryParseException { + if (p.getType() == FieldType.INTEGER) { + return intQuery(p); + } else if (p.getType() == FieldType.TIMESTAMP) { + return timestampQuery(p); + } else if (p.getType() == FieldType.EXACT) { + return exactQuery(p); + } else if (p.getType() == FieldType.PREFIX) { + return prefixQuery(p); + } else if (p.getType() == FieldType.FULL_TEXT) { + return fullTextQuery(p); + } else if (p instanceof SortKeyPredicate) { + return sortKeyQuery((SortKeyPredicate) p); + } else { + throw badFieldType(p.getType()); + } + } + + private static Term intTerm(String name, int value) { + BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT); + NumericUtils.intToPrefixCodedBytes(value, 0, bytes); + return new Term(name, bytes); + } + + private static Query intQuery(IndexPredicate p) + throws QueryParseException { + int value; + try { + // Can't use IntPredicate because it and IndexPredicate are different + // subclasses of OperatorPredicate. + value = Integer.valueOf(p.getValue()); + } catch (IllegalArgumentException e) { + throw new QueryParseException("not an integer: " + p.getValue()); + } + return new TermQuery(intTerm(p.getField().getName(), value)); + } + + private static Query sortKeyQuery(SortKeyPredicate p) { + return NumericRangeQuery.newLongRange( + p.getField().getName(), + p.getMinValue(), + p.getMaxValue(), + true, true); + } + + private static Query timestampQuery(IndexPredicate p) + throws QueryParseException { + if (p instanceof TimestampRangePredicate) { + TimestampRangePredicate r = + (TimestampRangePredicate) p; + return NumericRangeQuery.newIntRange( + r.getField().getName(), + toIndexTime(r.getMinTimestamp()), + toIndexTime(r.getMaxTimestamp()), + true, true); + } + throw new QueryParseException("not a timestamp: " + p); + } + + private static Query notTimestampQuery(TimestampRangePredicate r) + throws QueryParseException { + if (r.getMinTimestamp().getTime() == 0) { + return NumericRangeQuery.newIntRange( + r.getField().getName(), + toIndexTime(r.getMaxTimestamp()), + null, + true, true); + } + throw new QueryParseException("cannot negate: " + r); + } + + private static Query exactQuery(IndexPredicate p) { + if (p instanceof RegexPredicate) { + return regexQuery(p); + } else { + return new TermQuery(new Term(p.getField().getName(), p.getValue())); + } + } + + private static Query regexQuery(IndexPredicate p) { + String re = p.getValue(); + if (re.startsWith("^")) { + re = re.substring(1); + } + if (re.endsWith("$") && !re.endsWith("\\$")) { + re = re.substring(0, re.length() - 1); + } + return new RegexpQuery(new Term(p.getField().getName(), re)); + } + + private static Query prefixQuery(IndexPredicate p) { + return new PrefixQuery(new Term(p.getField().getName(), p.getValue())); + } + + private static Query fullTextQuery(IndexPredicate p) { + return new FuzzyQuery(new Term(p.getField().getName(), p.getValue())); + } + + public static int toIndexTime(Timestamp ts) { + return (int) (ts.getTime() / 60000); + } + + public static IllegalArgumentException badFieldType(FieldType t) { + return new IllegalArgumentException("unknown index field type " + t); + } + + private QueryBuilder() { + } +} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java index b783caf99a..60bc547e01 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java @@ -114,6 +114,10 @@ class SubIndex { return new NrtFuture(writer.deleteDocuments(term)); } + void deleteAll() throws IOException { + writer.deleteAll(); + } + IndexSearcher acquire() throws IOException { return nrtManager.acquire(); } diff --git a/gerrit-pgm/BUCK b/gerrit-pgm/BUCK index dd665dbd15..64653c024b 100644 --- a/gerrit-pgm/BUCK +++ b/gerrit-pgm/BUCK @@ -13,6 +13,7 @@ java_library2( '//gerrit-server:common_rules', '//gerrit-reviewdb:server', '//gerrit-server:server', + '//gerrit-solr:solr', '//gerrit-sshd:sshd', '//gerrit-util-cli:cli', '//lib:args4j', @@ -31,7 +32,7 @@ java_library2( '//lib/jgit:jgit', '//lib/log:api', '//lib/log:log4j', - '//lib:lucene-core', + '//lib/lucene:core', '//lib/mina:sshd', '//lib/prolog:prolog-cafe', ], diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java index f0fd802882..a10e7e495f 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java @@ -59,6 +59,7 @@ import com.google.gerrit.server.plugins.PluginModule; import com.google.gerrit.server.schema.SchemaVersionCheck; import com.google.gerrit.server.ssh.NoSshKeyCache; import com.google.gerrit.server.ssh.NoSshModule; +import com.google.gerrit.solr.SolrIndexModule; import com.google.gerrit.sshd.SshKeyCacheImpl; import com.google.gerrit.sshd.SshModule; import com.google.gerrit.sshd.commands.MasterCommandModule; @@ -253,11 +254,18 @@ public class Daemon extends SiteProgram { modules.add(new SmtpEmailSender.Module()); modules.add(new SignedTokenEmailTokenVerifier.Module()); modules.add(new PluginModule()); - if (IndexModule.isEnabled(cfgInjector)) { - modules.add(new LuceneIndexModule()); - } else { - modules.add(new NoIndexModule()); + AbstractModule changeIndexModule; + switch (IndexModule.getIndexType(cfgInjector)) { + case LUCENE: + changeIndexModule = new LuceneIndexModule(); + break; + case SOLR: + changeIndexModule = new SolrIndexModule(); + break; + default: + changeIndexModule = new NoIndexModule(); } + modules.add(changeIndexModule); if (httpd) { modules.add(new CanonicalWebUrlModule() { @Override diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java index 438d4b161a..19ae1d3c80 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java @@ -14,9 +14,6 @@ package com.google.gerrit.pgm; -import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS; -import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig; -import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION; import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER; import com.google.common.base.Stopwatch; @@ -41,16 +38,19 @@ import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.cache.CacheRemovalListener; import com.google.gerrit.server.cache.h2.DefaultCacheFactory; -import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.MultiProgressMonitor; import com.google.gerrit.server.git.MultiProgressMonitor.Task; +import com.google.gerrit.server.index.ChangeIndex; import com.google.gerrit.server.index.ChangeIndexer; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.index.IndexModule; +import com.google.gerrit.server.index.IndexModule.IndexType; +import com.google.gerrit.server.index.NoIndexModule; import com.google.gerrit.server.patch.PatchListCacheImpl; import com.google.gerrit.server.patch.PatchListLoader; import com.google.gerrit.server.query.change.ChangeData; +import com.google.gerrit.solr.SolrIndexModule; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.SchemaFactory; import com.google.inject.AbstractModule; @@ -61,8 +61,6 @@ import com.google.inject.Provider; import com.google.inject.ProvisionException; import com.google.inject.TypeLiteral; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffFormatter; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -78,14 +76,11 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.io.DisabledOutputStream; import org.kohsuke.args4j.Option; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.util.Collections; @@ -112,24 +107,24 @@ public class Reindex extends SiteProgram { private Injector dbInjector; private Injector sysInjector; - private SitePaths sitePaths; @Override public int run() throws Exception { mustHaveValidSite(); dbInjector = createDbInjector(MULTI_USER); - if (!IndexModule.isEnabled(dbInjector)) { - throw die("Secondary index not enabled"); + if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) { + throw die("index.type must be configured (or not SQL)"); } + LifecycleManager dbManager = new LifecycleManager(); dbManager.add(dbInjector); dbManager.start(); - sitePaths = dbInjector.getInstance(SitePaths.class); + + sysInjector = createSysInjector(); // Delete before any index may be created depending on this data. deleteAll(); - sysInjector = createSysInjector(); LifecycleManager sysManager = new LifecycleManager(); sysManager.add(sysInjector); sysManager.start(); @@ -145,7 +140,18 @@ public class Reindex extends SiteProgram { private Injector createSysInjector() { List modules = Lists.newArrayList(); modules.add(PatchListCacheImpl.module()); - modules.add(new LuceneIndexModule(false, threads, dryRun)); + AbstractModule changeIndexModule; + switch (IndexModule.getIndexType(dbInjector)) { + case LUCENE: + changeIndexModule = new LuceneIndexModule(false, threads, dryRun); + break; + case SOLR: + changeIndexModule = new SolrIndexModule(false, threads); + break; + default: + changeIndexModule = new NoIndexModule(); + } + modules.add(changeIndexModule); modules.add(new ReviewDbModule()); modules.add(new AbstractModule() { @SuppressWarnings("rawtypes") @@ -206,19 +212,8 @@ public class Reindex extends SiteProgram { if (dryRun) { return; } - for (String index : SCHEMA_VERSIONS.keySet()) { - File file = new File(sitePaths.index_dir, index); - if (file.exists()) { - Directory dir = FSDirectory.open(file); - try { - for (String name : dir.listAll()) { - dir.deleteFile(name); - } - } finally { - dir.close(); - } - } - } + ChangeIndex index = sysInjector.getInstance(ChangeIndex.class); + index.deleteAll(); } private int indexAll() throws Exception { @@ -480,14 +475,7 @@ public class Reindex extends SiteProgram { if (dryRun) { return; } - FileBasedConfig cfg = - new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect()); - cfg.load(); - - for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { - cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue()); - } - cfg.setEnum("lucene", null, "version", LUCENE_VERSION); - cfg.save(); + ChangeIndex index = sysInjector.getInstance(ChangeIndex.class); + index.finishIndex(); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java index cae5d7d8e0..3745eb5b19 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java @@ -21,6 +21,8 @@ import com.google.gerrit.server.query.QueryParseException; import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.ChangeDataSource; +import org.eclipse.jgit.errors.ConfigInvalidException; + import java.io.IOException; /** @@ -51,11 +53,21 @@ public interface ChangeIndex { return Futures.immediateFuture(null); } + @Override + public void deleteAll() throws IOException { + // Do nothing. + } + @Override public ChangeDataSource getSource(Predicate p) throws QueryParseException { throw new UnsupportedOperationException(); } + + @Override + public void finishIndex() { + // Do nothing. + } }; /** @@ -94,6 +106,13 @@ public interface ChangeIndex { */ public ListenableFuture delete(ChangeData cd) throws IOException; + /** + * Delete all change documents from the index. + * + * @throws IOException + */ + public void deleteAll() throws IOException; + /** * Convert the given operator predicate into a source searching the index and * returning only the documents matching that predicate. @@ -108,4 +127,13 @@ public interface ChangeIndex { */ public ChangeDataSource getSource(Predicate p) throws QueryParseException; + + /** + * Mark completion of indexing. + * + * @throws ConfigInvalidException + * @throws IOException + */ + public void finishIndex() throws IOException, + ConfigInvalidException; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java index 7230ad7405..5bdbd96a99 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java @@ -36,9 +36,15 @@ import org.eclipse.jgit.lib.Config; * implementations (e.g. Lucene). */ public class IndexModule extends AbstractModule { - public static boolean isEnabled(Injector injector) { - return injector.getInstance(Key.get(Config.class, GerritServerConfig.class)) - .getBoolean("index", null, "enabled", false); + public enum IndexType { + SQL, LUCENE, SOLR; + } + + /** Type of secondary index. */ + public static IndexType getIndexType(Injector injector) { + Config cfg = injector.getInstance( + Key.get(Config.class, GerritServerConfig.class)); + return cfg.getEnum("index", null, "type", IndexType.SQL); } private final int threads; diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/change/IndexRewriteTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/IndexRewriteTest.java index 5e605332bf..a754524761 100644 --- a/gerrit-server/src/test/java/com/google/gerrit/server/query/change/IndexRewriteTest.java +++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/IndexRewriteTest.java @@ -56,11 +56,21 @@ public class IndexRewriteTest extends TestCase { throw new UnsupportedOperationException(); } + @Override + public void deleteAll() { + throw new UnsupportedOperationException(); + } + @Override public ChangeDataSource getSource(Predicate p) throws QueryParseException { return new Source(); } + + @Override + public void finishIndex() { + throw new UnsupportedOperationException(); + } } private static class Source implements ChangeDataSource { diff --git a/gerrit-solr/BUCK b/gerrit-solr/BUCK new file mode 100644 index 0000000000..4176561510 --- /dev/null +++ b/gerrit-solr/BUCK @@ -0,0 +1,19 @@ +java_library( + name = 'solr', + srcs = glob(['src/main/java/**/*.java']), + deps = [ + '//gerrit-antlr:query_exception', + '//gerrit-extension-api:api', + '//gerrit-lucene:query_builder', + '//gerrit-reviewdb:client', + '//gerrit-server:server', + '//lib:guava', + '//lib:gwtorm', + '//lib/guice:guice', + '//lib/jgit:jgit', + '//lib/log:api', + '//lib/lucene:core', + '//lib/solr:solrj', + ], + visibility = ['PUBLIC'], +) diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java new file mode 100644 index 0000000000..67c4d0ed10 --- /dev/null +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java @@ -0,0 +1,80 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package com.google.gerrit.server.git; + +package com.google.gerrit.solr; + +import com.google.common.collect.ImmutableMap; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.ChangeField; +import com.google.inject.Inject; +import com.google.inject.ProvisionException; + +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +class IndexVersionCheck implements LifecycleListener { + public static final Map SCHEMA_VERSIONS = ImmutableMap.of( + SolrChangeIndex.CHANGES_OPEN, ChangeField.SCHEMA_VERSION, + SolrChangeIndex.CHANGES_CLOSED, ChangeField.SCHEMA_VERSION); + + public static File solrIndexConfig(SitePaths sitePaths) { + return new File(sitePaths.index_dir, "gerrit_index.config"); + } + + private final SitePaths sitePaths; + + @Inject + IndexVersionCheck(SitePaths sitePaths) { + this.sitePaths = sitePaths; + } + + @Override + public void start() { + // TODO Query schema version from a special meta-document + File file = solrIndexConfig(sitePaths); + try { + FileBasedConfig cfg = new FileBasedConfig(file, FS.detect()); + cfg.load(); + for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { + int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0); + if (schemaVersion != e.getValue()) { + throw new ProvisionException(String.format( + "wrong index schema version for \"%s\": expected %d, found %d%s", + e.getKey(), e.getValue(), schemaVersion, upgrade())); + } + } + } catch (IOException e) { + throw new ProvisionException("unable to read " + file); + } catch (ConfigInvalidException e) { + throw new ProvisionException("invalid config file " + file); + } + } + + @Override + public void stop() { + // Do nothing. + } + + private final String upgrade() { + return "\nRun reindex to rebuild the index:\n" + + "$ java -jar gerrit.war reindex -d " + + sitePaths.site_path.getAbsolutePath(); + } +} diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java new file mode 100644 index 0000000000..c28a0fc3e5 --- /dev/null +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java @@ -0,0 +1,318 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package com.google.gerrit.server.git; + +package com.google.gerrit.solr; + +import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES; +import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES; +import static com.google.gerrit.solr.IndexVersionCheck.SCHEMA_VERSIONS; +import static com.google.gerrit.solr.IndexVersionCheck.solrIndexConfig; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.lucene.QueryBuilder; +import com.google.gerrit.reviewdb.client.Change; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.ChangeField; +import com.google.gerrit.server.index.ChangeIndex; +import com.google.gerrit.server.index.FieldDef; +import com.google.gerrit.server.index.FieldDef.FillArgs; +import com.google.gerrit.server.index.FieldType; +import com.google.gerrit.server.query.Predicate; +import com.google.gerrit.server.query.QueryParseException; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.gerrit.server.query.change.ChangeDataSource; +import com.google.gerrit.server.query.change.IndexRewriteImpl; +import com.google.gwtorm.server.OrmException; +import com.google.gwtorm.server.ResultSet; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import org.apache.lucene.search.Query; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrServer; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Secondary index implementation using a remote Solr instance. */ +@Singleton +class SolrChangeIndex implements ChangeIndex, LifecycleListener { + public static final String CHANGES_OPEN = "changes_open"; + public static final String CHANGES_CLOSED = "changes_closed"; + private static final String ID_FIELD = ChangeField.LEGACY_ID.getName(); + + private final FillArgs fillArgs; + private final SitePaths sitePaths; + private final CloudSolrServer openIndex; + private final CloudSolrServer closedIndex; + + @Inject + SolrChangeIndex( + @GerritServerConfig Config cfg, + FillArgs fillArgs, + SitePaths sitePaths) throws IOException { + this.fillArgs = fillArgs; + this.sitePaths = sitePaths; + + String url = cfg.getString("index", "solr", "url"); + if (Strings.isNullOrEmpty(url)) { + throw new IllegalStateException("index.solr.url must be supplied"); + } + + openIndex = new CloudSolrServer(url); + openIndex.setDefaultCollection(CHANGES_OPEN); + + closedIndex = new CloudSolrServer(url); + closedIndex.setDefaultCollection(CHANGES_CLOSED); + } + + @Override + public void start() { + // Do nothing. + } + + @Override + public void stop() { + openIndex.shutdown(); + closedIndex.shutdown(); + } + + @Override + public ListenableFuture insert(ChangeData cd) throws IOException { + String id = cd.getId().toString(); + SolrInputDocument doc = toDocument(cd); + try { + if (cd.getChange().getStatus().isOpen()) { + closedIndex.deleteById(id); + openIndex.add(doc); + } else { + openIndex.deleteById(id); + closedIndex.add(doc); + } + } catch (SolrServerException e) { + throw new IOException(e); + } + commit(openIndex); + commit(closedIndex); + return Futures.immediateFuture(null); + } + + @Override + public ListenableFuture replace(ChangeData cd) throws IOException { + String id = cd.getId().toString(); + SolrInputDocument doc = toDocument(cd); + try { + if (cd.getChange().getStatus().isOpen()) { + closedIndex.deleteById(id); + openIndex.add(doc); + } else { + openIndex.deleteById(id); + closedIndex.add(doc); + } + } catch (SolrServerException e) { + throw new IOException(e); + } + commit(openIndex); + commit(closedIndex); + return Futures.immediateFuture(null); + } + + @Override + public ListenableFuture delete(ChangeData cd) throws IOException { + String id = cd.getId().toString(); + try { + if (cd.getChange().getStatus().isOpen()) { + openIndex.deleteById(id); + commit(openIndex); + } else { + closedIndex.deleteById(id); + commit(closedIndex); + } + return Futures.immediateFuture(null); + } catch (SolrServerException e) { + throw new IOException(e); + } + } + + @Override + public void deleteAll() throws IOException { + try { + openIndex.deleteByQuery("*:*"); + closedIndex.deleteByQuery("*:*"); + } catch (SolrServerException e) { + throw new IOException(e); + } + commit(openIndex); + commit(closedIndex); + } + + @Override + public ChangeDataSource getSource(Predicate p) + throws QueryParseException { + Set statuses = IndexRewriteImpl.getPossibleStatus(p); + List indexes = Lists.newArrayListWithCapacity(2); + if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) { + indexes.add(openIndex); + } + if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) { + indexes.add(closedIndex); + } + return new QuerySource(indexes, QueryBuilder.toQuery(p)); + } + + private void commit(SolrServer server) throws IOException { + try { + server.commit(); + } catch (SolrServerException e) { + throw new IOException(e); + } + } + + private class QuerySource implements ChangeDataSource { + private final List indexes; + private final SolrQuery query; + + public QuerySource(List indexes, Query q) { + this.indexes = indexes; + + query = new SolrQuery(q.toString()); + query.setParam("shards.tolerant", true); + query.setFields(ID_FIELD); + query.setSort( + ChangeField.UPDATED.getName(), + SolrQuery.ORDER.desc); + } + + @Override + public int getCardinality() { + return 10; // TODO: estimate from solr? + } + + @Override + public boolean hasChange() { + return false; + } + + @Override + public ResultSet read() throws OrmException { + try { + // TODO Sort documents during merge to select only top N. + SolrDocumentList docs = new SolrDocumentList(); + for (SolrServer index : indexes) { + docs.addAll(index.query(query).getResults()); + } + + List result = Lists.newArrayListWithCapacity(docs.size()); + for (SolrDocument doc : docs) { + Integer v = (Integer) doc.getFieldValue(ID_FIELD); + result.add(new ChangeData(new Change.Id(v.intValue()))); + } + + final List r = Collections.unmodifiableList(result); + return new ResultSet() { + @Override + public Iterator iterator() { + return r.iterator(); + } + + @Override + public List toList() { + return r; + } + + @Override + public void close() { + // Do nothing. + } + }; + } catch (SolrServerException e) { + throw new OrmException(e); + } + } + } + + private SolrInputDocument toDocument(ChangeData cd) throws IOException { + try { + SolrInputDocument result = new SolrInputDocument(); + for (FieldDef f : ChangeField.ALL.values()) { + if (f.isRepeatable()) { + add(result, f, (Iterable) f.get(cd, fillArgs)); + } else { + add(result, f, Collections.singleton(f.get(cd, fillArgs))); + } + } + return result; + } catch (OrmException e) { + throw new IOException(e); + } + } + + private void add(SolrInputDocument doc, FieldDef f, + Iterable values) throws OrmException { + if (f.getType() == FieldType.INTEGER) { + for (Object value : values) { + doc.addField(f.getName(), (Integer) value); + } + } else if (f.getType() == FieldType.LONG) { + for (Object value : values) { + doc.addField(f.getName(), (Long) value); + } + } else if (f.getType() == FieldType.TIMESTAMP) { + for (Object v : values) { + doc.addField(f.getName(), QueryBuilder.toIndexTime((Timestamp) v)); + } + } else if (f.getType() == FieldType.EXACT + || f.getType() == FieldType.PREFIX + || f.getType() == FieldType.FULL_TEXT) { + for (Object value : values) { + doc.addField(f.getName(), (String) value); + } + } else { + throw QueryBuilder.badFieldType(f.getType()); + } + } + + @Override + public void finishIndex() throws IOException, + ConfigInvalidException { + // TODO Move the schema version information to a special meta-document + FileBasedConfig cfg = new FileBasedConfig( + solrIndexConfig(sitePaths), + FS.detect()); + for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { + cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue()); + } + cfg.save(); + } +} diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java new file mode 100644 index 0000000000..4e1a548b40 --- /dev/null +++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java @@ -0,0 +1,43 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package com.google.gerrit.server.git; + +package com.google.gerrit.solr; + +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.server.index.ChangeIndex; +import com.google.gerrit.server.index.IndexModule; + +public class SolrIndexModule extends LifecycleModule { + private final boolean checkVersion; + private final int threads; + + public SolrIndexModule() { + this(true, 0); + } + + public SolrIndexModule(boolean checkVersion, int threads) { + this.checkVersion = checkVersion; + this.threads = threads; + } + + @Override + protected void configure() { + install(new IndexModule(threads)); + bind(ChangeIndex.class).to(SolrChangeIndex.class); + listener().to(SolrChangeIndex.class); + if (checkVersion) { + listener().to(IndexVersionCheck.class); + } + } +} diff --git a/gerrit-war/BUCK b/gerrit-war/BUCK index 1fef7f8b50..3f6cf48b66 100644 --- a/gerrit-war/BUCK +++ b/gerrit-war/BUCK @@ -10,6 +10,7 @@ java_library2( '//gerrit-reviewdb:server', '//gerrit-server:common_rules', '//gerrit-server:server', + '//gerrit-solr:solr', '//gerrit-sshd:sshd', '//lib:gwtorm', '//lib/guice:guice', diff --git a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java index 65bbfff51d..da08555e55 100644 --- a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java +++ b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java @@ -50,6 +50,7 @@ import com.google.gerrit.server.schema.DataSourceType; import com.google.gerrit.server.schema.DatabaseModule; import com.google.gerrit.server.schema.SchemaModule; import com.google.gerrit.server.schema.SchemaVersionCheck; +import com.google.gerrit.solr.SolrIndexModule; import com.google.gerrit.sshd.SshKeyCacheImpl; import com.google.gerrit.sshd.SshModule; import com.google.gerrit.sshd.commands.MasterCommandModule; @@ -238,11 +239,18 @@ public class WebAppInitializer extends GuiceServletContextListener { modules.add(new SmtpEmailSender.Module()); modules.add(new SignedTokenEmailTokenVerifier.Module()); modules.add(new PluginModule()); - if (IndexModule.isEnabled(cfgInjector)) { - modules.add(new LuceneIndexModule()); - } else { - modules.add(new NoIndexModule()); + AbstractModule changeIndexModule; + switch (IndexModule.getIndexType(cfgInjector)) { + case LUCENE: + changeIndexModule = new LuceneIndexModule(); + break; + case SOLR: + changeIndexModule = new SolrIndexModule(); + break; + default: + changeIndexModule = new NoIndexModule(); } + modules.add(changeIndexModule); modules.add(new CanonicalWebUrlModule() { @Override protected Class> provider() { diff --git a/lib/BUCK b/lib/BUCK index 5f55f70d67..a9d0bf6c3f 100644 --- a/lib/BUCK +++ b/lib/BUCK @@ -246,27 +246,3 @@ maven_jar( visibility = ['//lib:easymock'], attach_source = False, ) - -maven_jar( - name = 'lucene-core', - id = 'org.apache.lucene:lucene-core:4.3.0', - bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3', - src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a', - license = 'Apache2.0', - exclude = [ - 'META-INF/LICENSE.txt', - 'META-INF/NOTICE.txt', - ], -) - -maven_jar( - name = 'lucene-analyzers-common', - id = 'org.apache.lucene:lucene-analyzers-common:4.3.0', - bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56', - src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b', - license = 'Apache2.0', - exclude = [ - 'META-INF/LICENSE.txt', - 'META-INF/NOTICE.txt', - ], -) diff --git a/lib/commons/BUCK b/lib/commons/BUCK index e5c78abcd6..6f412e4a02 100644 --- a/lib/commons/BUCK +++ b/lib/commons/BUCK @@ -15,7 +15,10 @@ maven_jar( license = 'Apache2.0', exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'], attach_source = False, - visibility = ['//lib:velocity'], + visibility = [ + '//lib:velocity', + '//lib/solr:zookeeper', + ], ) maven_jar( @@ -39,19 +42,6 @@ maven_jar( exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'], ) -maven_jar( - name = 'logging', - id = 'commons-logging:commons-logging:1.1.1', - sha1 = '5043bfebc3db072ed80fbd362e7caf00e885d8ae', - license = 'Apache2.0', - exclude = [ - 'META-INF/LICENSE', - 'META-INF/NOTICE', - ], - attach_source = False, - visibility = ['//lib/openid:'], -) - maven_jar( name = 'net', id = 'commons-net:commons-net:2.2', @@ -77,3 +67,39 @@ maven_jar( attach_source = False, exclude = ['META-INF/LICENSE'], ) + +maven_jar( + name = 'io', + id = 'commons-io:commons-io:1.4', + sha1 = 'a8762d07e76cfde2395257a5da47ba7c1dbd3dce', + license = 'Apache2.0', +) + +maven_jar( + name = 'httpclient', + id = 'org.apache.httpcomponents:httpclient:4.2.5', + bin_sha1 = '666e26e76f2e87d84e4f16acb546481ae1b8e9a6', + src_sha1 = '55d345272944d7e8dace47925336a3764ee0e24b', + license = 'Apache2.0', + deps = [ + ':codec', + ':httpcore', + '//lib/log:jcl-over-slf4j', + ], +) + +maven_jar( + name = 'httpcore', + id = 'org.apache.httpcomponents:httpcore:4.2.4', + bin_sha1 = '3b7f38df6de5dd8b500e602ae8c2dd5ee446f883', + src_sha1 = 'c3ffe3a73348645042fb0b06303b6a3de194494e', + license = 'Apache2.0', +) + +maven_jar( + name = 'httpmime', + id = 'org.apache.httpcomponents:httpmime:4.2.5', + bin_sha1 = '412b9914d0adec6d5716df1ada8acbc4f6f2dd37', + src_sha1 = 'c07ce7f6b153284a9ebaf58532c2442200cf3aa2', + license = 'Apache2.0', +) diff --git a/lib/log/BUCK b/lib/log/BUCK index 5b541c42b1..2659fcdf25 100644 --- a/lib/log/BUCK +++ b/lib/log/BUCK @@ -22,3 +22,10 @@ maven_jar( license = 'Apache2.0', exclude = ['META-INF/LICENSE', 'META-INF/NOTICE'], ) + +maven_jar( + name = 'jcl-over-slf4j', + id = 'org.slf4j:jcl-over-slf4j:1.6.1', + sha1 = '99c61095a14dfc9e47a086068033c286bf236475', + license = 'slf4j', +) diff --git a/lib/lucene/BUCK b/lib/lucene/BUCK new file mode 100644 index 0000000000..e0972d9e8c --- /dev/null +++ b/lib/lucene/BUCK @@ -0,0 +1,49 @@ +include_defs('//lib/maven.defs') + +maven_jar( + name = 'core', + id = 'org.apache.lucene:lucene-core:4.3.0', + bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3', + src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a', + license = 'Apache2.0', + exclude = [ + 'META-INF/LICENSE.txt', + 'META-INF/NOTICE.txt', + ], +) + +maven_jar( + name = 'analyzers-common', + id = 'org.apache.lucene:lucene-analyzers-common:4.3.0', + bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56', + src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b', + license = 'Apache2.0', + exclude = [ + 'META-INF/LICENSE.txt', + 'META-INF/NOTICE.txt', + ], +) + +maven_jar( + name = 'highlighter', + id = 'org.apache.lucene:lucene-highlighter:4.3.0', + bin_sha1 = '9e6d60921e16a0d6b2e609c6a02a8b08cd7f643c', + src_sha1 = '0ff70cae1a8fb7af29bf254d90e9885961deed5e', + license = 'Apache2.0', +) + +maven_jar( + name = 'queries', + id = 'org.apache.lucene:lucene-queries:4.3.0', + bin_sha1 = '68e01022bdf4f869b95362c9af964846e5d3cf2d', + src_sha1 = '3e3541c1b9f44c532ce88ab6a12216566c3399df', + license = 'Apache2.0', +) + +maven_jar( + name = 'spellchecker', + id = 'org.apache.lucene:lucene-spellchecker:3.6.2', + bin_sha1 = '15db0c0cfee44e275f15ad046e46b9a05910ad24', + src_sha1 = 'bbecb3fb725ae594101c165a72c102296007c203', + license = 'Apache2.0', +) diff --git a/lib/openid/BUCK b/lib/openid/BUCK index b766532c6c..c6c8baf6ae 100644 --- a/lib/openid/BUCK +++ b/lib/openid/BUCK @@ -6,10 +6,10 @@ maven_jar( sha1 = 'de4f1b33d3b0f0b2ab1d32834ec1190b39db4160', license = 'Apache2.0', deps = [ - ':httpclient', ':nekohtml', ':xerces', - '//lib/commons:logging', + '//lib/commons:httpclient', + '//lib/log:jcl-over-slf4j', '//lib/guice:guice', ], visibility = ['PUBLIC'], @@ -33,31 +33,3 @@ maven_jar( attach_source = False, visibility = [], ) - -maven_jar( - name = 'httpclient', - id = 'org.apache.httpcomponents:httpclient:4.1', - sha1 = '93cd011acb220de08b57d96106e5800d7097742b', - license = 'Apache2.0', - deps = [ - ':httpcore', - '//lib/commons:codec', - '//lib/commons:logging', - ], - exclude = [ - 'META-INF/LICENSE.txt', - 'META-INF/NOTICE.txt', - ], - visibility = ['//gerrit-acceptance-tests:'], -) - -maven_jar( - name = 'httpcore', - id = 'org.apache.httpcomponents:httpcore:4.1', - sha1 = '33fc26c02f8043ab0ede19eadc8c9885386b255c', - license = 'Apache2.0', - exclude = [ - 'META-INF/LICENSE.txt', - 'META-INF/NOTICE.txt', - ], -) diff --git a/lib/solr/BUCK b/lib/solr/BUCK new file mode 100644 index 0000000000..afaa948511 --- /dev/null +++ b/lib/solr/BUCK @@ -0,0 +1,33 @@ +include_defs('//lib/maven.defs') + +# Java client library to use Solr over the network. +maven_jar( + name = 'solrj', + id = 'org.apache.solr:solr-solrj:4.3.1', + sha1 = '433fe37796e67eaeb4452f69eb1fae2de27cb7a8', + license = 'Apache2.0', + deps = [ + ':noggit', + ':zookeeper', + '//lib/commons:httpclient', + '//lib/commons:httpmime', + '//lib/commons:io', + ], +) + +maven_jar( + name = 'noggit', + id = 'org.noggit:noggit:0.5', + sha1 = '8e6e65624d2e09a30190c6434abe23b7d4e5413c', + license = 'Apache2.0', + visibility = [], +) + +maven_jar( + name = 'zookeeper', + id = 'org.apache.zookeeper:zookeeper:3.4.5', + sha1 = 'c0f69fb36526552a8f0bc548a6c33c49cf08e562', + license = 'Apache2.0', + deps = ['//lib/log:api'], + visibility = [], +)