opendev/gerrit
Code Issues Proposed changes

Add secondary index implementation using SolrCloud

Browse Source 
SolrCloud can be used instead of Lucene by adding "type = SOLR"
under [index] and "url = <zookeeper-url>" under [index "solr"]
in gerrit.config.

Change-Id: I0ff8579c5e23c58b16f3605bc20eba4e80fb40fc
This commit is contained in:
Ahaan Ugale 2013-06-12 17:22:19 -06:00 committed by Shawn Pearce
parent 9279b29da7
commit 404c8246bc
23 changed files with 940 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
gerrit-acceptance-tests/BUCK
View File

@ -24,13 +24,13 @@ java_test(
'//lib:junit',
'//lib:servlet-api-3_0',
'//lib/commons:httpclient',
'//lib/commons:httpcore',
'//lib/log:impl_log4j',
'//lib/log:log4j',
'//lib/guice:guice',
'//lib/jgit:jgit',
'//lib/jgit:junit',
'//lib/openid:httpclient',
'//lib/openid:httpcore',
],
source_under_test = TEST,
labels = ['slow'],

24
gerrit-lucene/BUCK
View File

@ -1,18 +1,36 @@
QUERY_BUILDER = [
'src/main/java/com/google/gerrit/lucene/QueryBuilder.java',
]
java_library(
name = 'query_builder',
srcs = QUERY_BUILDER,
deps = [
'//gerrit-antlr:query_exception',
'//gerrit-reviewdb:server',
'//gerrit-server:server',
'//lib:gwtorm',
'//lib/lucene:core',
],
visibility = ['PUBLIC'],
)
java_library(
name = 'lucene',
srcs = glob(['src/main/java/**/*.java']),
srcs = glob(['src/main/java/**/*.java'], excludes = QUERY_BUILDER),
deps = [
':query_builder',
'//gerrit-antlr:query_exception',
'//gerrit-extension-api:api',
'//gerrit-reviewdb:server',
'//gerrit-server:server',
'//lib:guava',
'//lib:gwtorm',
'//lib:lucene-analyzers-common',
'//lib:lucene-core',
'//lib/guice:guice',
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:analyzers-common',
'//lib/lucene:core',
],
visibility = ['PUBLIC'],
)

193
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@ -14,11 +14,10 @@
package com.google.gerrit.lucene;
import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
@ -35,18 +34,11 @@ import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.RegexPredicate;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.query.AndPredicate;
import com.google.gerrit.server.query.NotPredicate;
import com.google.gerrit.server.query.OrPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gerrit.server.query.change.IndexRewriteImpl;
import com.google.gerrit.server.query.change.SortKeyPredicate;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
@ -62,24 +54,18 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Version;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -89,6 +75,7 @@ import java.sql.Timestamp;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@ -122,6 +109,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
return writerConfig;
}
private final SitePaths sitePaths;
private final FillArgs fillArgs;
private final ExecutorService executor;
private final boolean readOnly;
@ -131,6 +119,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
LuceneChangeIndex(Config cfg, SitePaths sitePaths,
ListeningScheduledExecutorService executor, FillArgs fillArgs,
boolean readOnly) throws IOException {
this.sitePaths = sitePaths;
this.fillArgs = fillArgs;
this.executor = executor;
this.readOnly = readOnly;
@ -167,7 +156,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
@SuppressWarnings("unchecked")
@Override
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
Term id = idTerm(cd);
Term id = QueryBuilder.idTerm(cd);
Document doc = toDocument(cd);
if (readOnly) {
return Futures.immediateFuture(null);
@ -187,7 +176,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
@SuppressWarnings("unchecked")
@Override
public ListenableFuture<Void> replace(ChangeData cd) throws IOException {
Term id = idTerm(cd);
Term id = QueryBuilder.idTerm(cd);
Document doc = toDocument(cd);
if (readOnly) {
return Futures.immediateFuture(null);
@ -206,7 +195,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
@SuppressWarnings("unchecked")
@Override
public ListenableFuture<Void> delete(ChangeData cd) throws IOException {
Term id = idTerm(cd);
Term id = QueryBuilder.idTerm(cd);
if (readOnly) {
return Futures.immediateFuture(null);
}
@ -226,6 +215,11 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
});
}
@Override
public void deleteAll() throws IOException {
openIndex.deleteAll();
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p)
throws QueryParseException {
@ -237,138 +231,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(closedIndex);
}
return new QuerySource(indexes, toQuery(p));
}
private Term idTerm(ChangeData cd) {
return intTerm(ID_FIELD, cd.getId().get());
}
private Query toQuery(Predicate<ChangeData> p) throws QueryParseException {
if (p.getClass() == AndPredicate.class) {
return booleanQuery(p, MUST);
} else if (p.getClass() == OrPredicate.class) {
return booleanQuery(p, SHOULD);
} else if (p.getClass() == NotPredicate.class) {
if (p.getChild(0) instanceof TimestampRangePredicate) {
return notTimestampQuery(
(TimestampRangePredicate<ChangeData>) p.getChild(0));
}
return booleanQuery(p, MUST_NOT);
} else if (p instanceof IndexPredicate) {
return fieldQuery((IndexPredicate<ChangeData>) p);
} else {
throw new QueryParseException("Cannot convert to index predicate: " + p);
}
}
private Query booleanQuery(Predicate<ChangeData> p, BooleanClause.Occur o)
throws QueryParseException {
BooleanQuery q = new BooleanQuery();
for (int i = 0; i < p.getChildCount(); i++) {
q.add(toQuery(p.getChild(i)), o);
}
return q;
}
private Query fieldQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
if (p.getType() == FieldType.INTEGER) {
return intQuery(p);
} else if (p.getType() == FieldType.TIMESTAMP) {
return timestampQuery(p);
} else if (p.getType() == FieldType.EXACT) {
return exactQuery(p);
} else if (p.getType() == FieldType.PREFIX) {
return prefixQuery(p);
} else if (p.getType() == FieldType.FULL_TEXT) {
return fullTextQuery(p);
} else if (p instanceof SortKeyPredicate) {
return sortKeyQuery((SortKeyPredicate) p);
} else {
throw badFieldType(p.getType());
}
}
private Term intTerm(String name, int value) {
BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
NumericUtils.intToPrefixCodedBytes(value, 0, bytes);
return new Term(name, bytes);
}
private Query intQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
int value;
try {
// Can't use IntPredicate because it and IndexPredicate are different
// subclasses of OperatorPredicate.
value = Integer.valueOf(p.getValue());
} catch (IllegalArgumentException e) {
throw new QueryParseException("not an integer: " + p.getValue());
}
return new TermQuery(intTerm(p.getField().getName(), value));
}
private static Query sortKeyQuery(SortKeyPredicate p) {
return NumericRangeQuery.newLongRange(
p.getField().getName(),
p.getMinValue(),
p.getMaxValue(),
true, true);
}
private static Query timestampQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<ChangeData> r =
(TimestampRangePredicate<ChangeData>) p;
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMinTimestamp()),
toIndexTime(r.getMaxTimestamp()),
true, true);
}
throw new QueryParseException("not a timestamp: " + p);
}
private static Query notTimestampQuery(TimestampRangePredicate<ChangeData> r)
throws QueryParseException {
if (r.getMinTimestamp().getTime() == 0) {
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMaxTimestamp()),
null,
true, true);
}
throw new QueryParseException("cannot negate: " + r);
}
private Query exactQuery(IndexPredicate<ChangeData> p) {
if (p instanceof RegexPredicate<?>) {
return regexQuery(p);
} else {
return new TermQuery(new Term(p.getField().getName(), p.getValue()));
}
}
private Query regexQuery(IndexPredicate<ChangeData> p) {
String re = p.getValue();
if (re.startsWith("^")) {
re = re.substring(1);
}
if (re.endsWith("$") && !re.endsWith("\\$")) {
re = re.substring(0, re.length() - 1);
}
return new RegexpQuery(new Term(p.getField().getName(), re));
}
private Query prefixQuery(IndexPredicate<ChangeData> p) {
return new PrefixQuery(new Term(p.getField().getName(), p.getValue()));
}
private Query fullTextQuery(IndexPredicate<ChangeData> p) {
return new FuzzyQuery(new Term(p.getField().getName(), p.getValue()));
return new QuerySource(indexes, QueryBuilder.toQuery(p));
}
private static class QuerySource implements ChangeDataSource {
@ -485,7 +348,8 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
}
} else if (f.getType() == FieldType.TIMESTAMP) {
for (Object v : values) {
doc.add(new IntField(name, toIndexTime((Timestamp) v), store));
int t = QueryBuilder.toIndexTime((Timestamp) v);
doc.add(new IntField(name, t, store));
}
} else if (f.getType() == FieldType.EXACT
|| f.getType() == FieldType.PREFIX) {
@ -497,19 +361,24 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
doc.add(new TextField(name, (String) value, store));
}
} else {
throw badFieldType(f.getType());
throw QueryBuilder.badFieldType(f.getType());
}
}
private static int toIndexTime(Timestamp ts) {
return (int) (ts.getTime() / 60000);
}
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
private static IllegalArgumentException badFieldType(FieldType<?> t) {
return new IllegalArgumentException("unknown index field type " + t);
@Override
public void finishIndex() throws IOException,
ConfigInvalidException {
FileBasedConfig cfg =
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
}
cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
cfg.save();
}
}

192
gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java Normal file
View File

@ -0,0 +1,192 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.RegexPredicate;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.query.AndPredicate;
import com.google.gerrit.server.query.NotPredicate;
import com.google.gerrit.server.query.OrPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.SortKeyPredicate;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import java.sql.Timestamp;
public class QueryBuilder {
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
public static Term idTerm(ChangeData cd) {
return intTerm(ID_FIELD, cd.getId().get());
}
public static Query toQuery(Predicate<ChangeData> p)
throws QueryParseException {
if (p.getClass() == AndPredicate.class) {
return booleanQuery(p, MUST);
} else if (p.getClass() == OrPredicate.class) {
return booleanQuery(p, SHOULD);
} else if (p.getClass() == NotPredicate.class) {
if (p.getChild(0) instanceof TimestampRangePredicate) {
return notTimestampQuery(
(TimestampRangePredicate<ChangeData>) p.getChild(0));
}
return booleanQuery(p, MUST_NOT);
} else if (p instanceof IndexPredicate) {
return fieldQuery((IndexPredicate<ChangeData>) p);
} else {
throw new QueryParseException("Cannot convert to index predicate: " + p);
}
}
private static Query booleanQuery(Predicate<ChangeData> p, BooleanClause.Occur o)
throws QueryParseException {
BooleanQuery q = new BooleanQuery();
for (int i = 0; i < p.getChildCount(); i++) {
q.add(toQuery(p.getChild(i)), o);
}
return q;
}
private static Query fieldQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
if (p.getType() == FieldType.INTEGER) {
return intQuery(p);
} else if (p.getType() == FieldType.TIMESTAMP) {
return timestampQuery(p);
} else if (p.getType() == FieldType.EXACT) {
return exactQuery(p);
} else if (p.getType() == FieldType.PREFIX) {
return prefixQuery(p);
} else if (p.getType() == FieldType.FULL_TEXT) {
return fullTextQuery(p);
} else if (p instanceof SortKeyPredicate) {
return sortKeyQuery((SortKeyPredicate) p);
} else {
throw badFieldType(p.getType());
}
}
private static Term intTerm(String name, int value) {
BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
NumericUtils.intToPrefixCodedBytes(value, 0, bytes);
return new Term(name, bytes);
}
private static Query intQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
int value;
try {
// Can't use IntPredicate because it and IndexPredicate are different
// subclasses of OperatorPredicate.
value = Integer.valueOf(p.getValue());
} catch (IllegalArgumentException e) {
throw new QueryParseException("not an integer: " + p.getValue());
}
return new TermQuery(intTerm(p.getField().getName(), value));
}
private static Query sortKeyQuery(SortKeyPredicate p) {
return NumericRangeQuery.newLongRange(
p.getField().getName(),
p.getMinValue(),
p.getMaxValue(),
true, true);
}
private static Query timestampQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<ChangeData> r =
(TimestampRangePredicate<ChangeData>) p;
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMinTimestamp()),
toIndexTime(r.getMaxTimestamp()),
true, true);
}
throw new QueryParseException("not a timestamp: " + p);
}
private static Query notTimestampQuery(TimestampRangePredicate<ChangeData> r)
throws QueryParseException {
if (r.getMinTimestamp().getTime() == 0) {
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMaxTimestamp()),
null,
true, true);
}
throw new QueryParseException("cannot negate: " + r);
}
private static Query exactQuery(IndexPredicate<ChangeData> p) {
if (p instanceof RegexPredicate<?>) {
return regexQuery(p);
} else {
return new TermQuery(new Term(p.getField().getName(), p.getValue()));
}
}
private static Query regexQuery(IndexPredicate<ChangeData> p) {
String re = p.getValue();
if (re.startsWith("^")) {
re = re.substring(1);
}
if (re.endsWith("$") && !re.endsWith("\\$")) {
re = re.substring(0, re.length() - 1);
}
return new RegexpQuery(new Term(p.getField().getName(), re));
}
private static Query prefixQuery(IndexPredicate<ChangeData> p) {
return new PrefixQuery(new Term(p.getField().getName(), p.getValue()));
}
private static Query fullTextQuery(IndexPredicate<ChangeData> p) {
return new FuzzyQuery(new Term(p.getField().getName(), p.getValue()));
}
public static int toIndexTime(Timestamp ts) {
return (int) (ts.getTime() / 60000);
}
public static IllegalArgumentException badFieldType(FieldType<?> t) {
return new IllegalArgumentException("unknown index field type " + t);
}
private QueryBuilder() {
}
}

4
gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
View File

@ -114,6 +114,10 @@ class SubIndex {
return new NrtFuture(writer.deleteDocuments(term));
}
void deleteAll() throws IOException {
writer.deleteAll();
}
IndexSearcher acquire() throws IOException {
return nrtManager.acquire();
}

3
gerrit-pgm/BUCK
View File

@ -13,6 +13,7 @@ java_library2(
'//gerrit-server:common_rules',
'//gerrit-reviewdb:server',
'//gerrit-server:server',
'//gerrit-solr:solr',
'//gerrit-sshd:sshd',
'//gerrit-util-cli:cli',
'//lib:args4j',
@ -31,7 +32,7 @@ java_library2(
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/log:log4j',
'//lib:lucene-core',
'//lib/lucene:core',
'//lib/mina:sshd',
'//lib/prolog:prolog-cafe',
],

16
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java
View File

@ -59,6 +59,7 @@ import com.google.gerrit.server.plugins.PluginModule;
import com.google.gerrit.server.schema.SchemaVersionCheck;
import com.google.gerrit.server.ssh.NoSshKeyCache;
import com.google.gerrit.server.ssh.NoSshModule;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.gerrit.sshd.SshKeyCacheImpl;
import com.google.gerrit.sshd.SshModule;
import com.google.gerrit.sshd.commands.MasterCommandModule;
@ -253,11 +254,18 @@ public class Daemon extends SiteProgram {
modules.add(new SmtpEmailSender.Module());
modules.add(new SignedTokenEmailTokenVerifier.Module());
modules.add(new PluginModule());
if (IndexModule.isEnabled(cfgInjector)) {
modules.add(new LuceneIndexModule());
} else {
modules.add(new NoIndexModule());
AbstractModule changeIndexModule;
switch (IndexModule.getIndexType(cfgInjector)) {
case LUCENE:
changeIndexModule = new LuceneIndexModule();
break;
case SOLR:
changeIndexModule = new SolrIndexModule();
break;
default:
changeIndexModule = new NoIndexModule();
}
modules.add(changeIndexModule);
if (httpd) {
modules.add(new CanonicalWebUrlModule() {
@Override

62
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
View File

@ -14,9 +14,6 @@
package com.google.gerrit.pgm;
import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER;
import com.google.common.base.Stopwatch;
@ -41,16 +38,19 @@ import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.cache.CacheRemovalListener;
import com.google.gerrit.server.cache.h2.DefaultCacheFactory;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.MultiProgressMonitor;
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeIndexer;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.NoIndexModule;
import com.google.gerrit.server.patch.PatchListCacheImpl;
import com.google.gerrit.server.patch.PatchListLoader;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.AbstractModule;
@ -61,8 +61,6 @@ import com.google.inject.Provider;
import com.google.inject.ProvisionException;
import com.google.inject.TypeLiteral;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.errors.ConfigInvalidException;
@ -78,14 +76,11 @@ import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.io.DisabledOutputStream;
import org.kohsuke.args4j.Option;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Collections;
@ -108,24 +103,24 @@ public class Reindex extends SiteProgram {
private Injector dbInjector;
private Injector sysInjector;
private SitePaths sitePaths;
@Override
public int run() throws Exception {
mustHaveValidSite();
dbInjector = createDbInjector(MULTI_USER);
if (!IndexModule.isEnabled(dbInjector)) {
throw die("Secondary index not enabled");
if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) {
throw die("index.type must be configured (or not SQL)");
}
LifecycleManager dbManager = new LifecycleManager();
dbManager.add(dbInjector);
dbManager.start();
sitePaths = dbInjector.getInstance(SitePaths.class);
sysInjector = createSysInjector();
// Delete before any index may be created depending on this data.
deleteAll();
sysInjector = createSysInjector();
LifecycleManager sysManager = new LifecycleManager();
sysManager.add(sysInjector);
sysManager.start();
@ -141,7 +136,18 @@ public class Reindex extends SiteProgram {
private Injector createSysInjector() {
List<Module> modules = Lists.newArrayList();
modules.add(PatchListCacheImpl.module());
modules.add(new LuceneIndexModule(false, threads, dryRun));
AbstractModule changeIndexModule;
switch (IndexModule.getIndexType(dbInjector)) {
case LUCENE:
changeIndexModule = new LuceneIndexModule(false, threads, dryRun);
break;
case SOLR:
changeIndexModule = new SolrIndexModule(false, threads);
break;
default:
changeIndexModule = new NoIndexModule();
}
modules.add(changeIndexModule);
modules.add(new ReviewDbModule());
modules.add(new AbstractModule() {
@SuppressWarnings("rawtypes")
@ -202,19 +208,8 @@ public class Reindex extends SiteProgram {
if (dryRun) {
return;
}
for (String index : SCHEMA_VERSIONS.keySet()) {
File file = new File(sitePaths.index_dir, index);
if (file.exists()) {
Directory dir = FSDirectory.open(file);
try {
for (String name : dir.listAll()) {
dir.deleteFile(name);
}
} finally {
dir.close();
}
}
}
ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
index.deleteAll();
}
private int indexAll() throws Exception {
@ -445,14 +440,7 @@ public class Reindex extends SiteProgram {
if (dryRun) {
return;
}
FileBasedConfig cfg =
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
cfg.load();
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
}
cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
cfg.save();
ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
index.finishIndex();
}
}

28
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java
View File

@ -21,6 +21,8 @@ import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import org.eclipse.jgit.errors.ConfigInvalidException;
import java.io.IOException;
/**
@ -51,11 +53,21 @@ public interface ChangeIndex {
return Futures.immediateFuture(null);
}
@Override
public void deleteAll() throws IOException {
// Do nothing.
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p)
throws QueryParseException {
throw new UnsupportedOperationException();
}
@Override
public void finishIndex() {
// Do nothing.
}
};
/**
@ -94,6 +106,13 @@ public interface ChangeIndex {
*/
public ListenableFuture<Void> delete(ChangeData cd) throws IOException;
/**
* Delete all change documents from the index.
*
* @throws IOException
*/
public void deleteAll() throws IOException;
/**
* Convert the given operator predicate into a source searching the index and
* returning only the documents matching that predicate.
@ -108,4 +127,13 @@ public interface ChangeIndex {
*/
public ChangeDataSource getSource(Predicate<ChangeData> p)
throws QueryParseException;
/**
* Mark completion of indexing.
*
* @throws ConfigInvalidException
* @throws IOException
*/
public void finishIndex() throws IOException,
ConfigInvalidException;
}

12
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
View File

@ -36,9 +36,15 @@ import org.eclipse.jgit.lib.Config;
* implementations (e.g. Lucene).
*/
public class IndexModule extends AbstractModule {
public static boolean isEnabled(Injector injector) {
return injector.getInstance(Key.get(Config.class, GerritServerConfig.class))
.getBoolean("index", null, "enabled", false);
public enum IndexType {
SQL, LUCENE, SOLR;
}
/** Type of secondary index. */
public static IndexType getIndexType(Injector injector) {
Config cfg = injector.getInstance(
Key.get(Config.class, GerritServerConfig.class));
return cfg.getEnum("index", null, "type", IndexType.SQL);
}
private final int threads;

10
gerrit-server/src/test/java/com/google/gerrit/server/query/change/IndexRewriteTest.java
View File

@ -56,11 +56,21 @@ public class IndexRewriteTest extends TestCase {
throw new UnsupportedOperationException();
}
@Override
public void deleteAll() {
throw new UnsupportedOperationException();
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p)
throws QueryParseException {
return new Source();
}
@Override
public void finishIndex() {
throw new UnsupportedOperationException();
}
}
private static class Source implements ChangeDataSource {

19
gerrit-solr/BUCK Normal file
View File

@ -0,0 +1,19 @@
java_library(
name = 'solr',
srcs = glob(['src/main/java/**/*.java']),
deps = [
'//gerrit-antlr:query_exception',
'//gerrit-extension-api:api',
'//gerrit-lucene:query_builder',
'//gerrit-reviewdb:client',
'//gerrit-server:server',
'//lib:guava',
'//lib:gwtorm',
'//lib/guice:guice',
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:core',
'//lib/solr:solrj',
],
visibility = ['PUBLIC'],
)

80
gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java Normal file
View File

@ -0,0 +1,80 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.solr;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeField;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import java.io.File;
import java.io.IOException;
import java.util.Map;
class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
SolrChangeIndex.CHANGES_OPEN, ChangeField.SCHEMA_VERSION,
SolrChangeIndex.CHANGES_CLOSED, ChangeField.SCHEMA_VERSION);
public static File solrIndexConfig(SitePaths sitePaths) {
return new File(sitePaths.index_dir, "gerrit_index.config");
}
private final SitePaths sitePaths;
@Inject
IndexVersionCheck(SitePaths sitePaths) {
this.sitePaths = sitePaths;
}
@Override
public void start() {
// TODO Query schema version from a special meta-document
File file = solrIndexConfig(sitePaths);
try {
FileBasedConfig cfg = new FileBasedConfig(file, FS.detect());
cfg.load();
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
if (schemaVersion != e.getValue()) {
throw new ProvisionException(String.format(
"wrong index schema version for \"%s\": expected %d, found %d%s",
e.getKey(), e.getValue(), schemaVersion, upgrade()));
}
}
} catch (IOException e) {
throw new ProvisionException("unable to read " + file);
} catch (ConfigInvalidException e) {
throw new ProvisionException("invalid config file " + file);
}
}
@Override
public void stop() {
// Do nothing.
}
private final String upgrade() {
return "\nRun reindex to rebuild the index:\n"
+ "$ java -jar gerrit.war reindex -d "
+ sitePaths.site_path.getAbsolutePath();
}
}

318
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java Normal file
View File

@ -0,0 +1,318 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.solr;
import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES;
import static com.google.gerrit.solr.IndexVersionCheck.SCHEMA_VERSIONS;
import static com.google.gerrit.solr.IndexVersionCheck.solrIndexConfig;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lucene.QueryBuilder;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gerrit.server.query.change.IndexRewriteImpl;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.lucene.search.Query;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Secondary index implementation using a remote Solr instance. */
@Singleton
class SolrChangeIndex implements ChangeIndex, LifecycleListener {
public static final String CHANGES_OPEN = "changes_open";
public static final String CHANGES_CLOSED = "changes_closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
private final FillArgs fillArgs;
private final SitePaths sitePaths;
private final CloudSolrServer openIndex;
private final CloudSolrServer closedIndex;
@Inject
SolrChangeIndex(
@GerritServerConfig Config cfg,
FillArgs fillArgs,
SitePaths sitePaths) throws IOException {
this.fillArgs = fillArgs;
this.sitePaths = sitePaths;
String url = cfg.getString("index", "solr", "url");
if (Strings.isNullOrEmpty(url)) {
throw new IllegalStateException("index.solr.url must be supplied");
}
openIndex = new CloudSolrServer(url);
openIndex.setDefaultCollection(CHANGES_OPEN);
closedIndex = new CloudSolrServer(url);
closedIndex.setDefaultCollection(CHANGES_CLOSED);
}
@Override
public void start() {
// Do nothing.
}
@Override
public void stop() {
openIndex.shutdown();
closedIndex.shutdown();
}
@Override
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
String id = cd.getId().toString();
SolrInputDocument doc = toDocument(cd);
try {
if (cd.getChange().getStatus().isOpen()) {
closedIndex.deleteById(id);
openIndex.add(doc);
} else {
openIndex.deleteById(id);
closedIndex.add(doc);
}
} catch (SolrServerException e) {
throw new IOException(e);
}
commit(openIndex);
commit(closedIndex);
return Futures.immediateFuture(null);
}
@Override
public ListenableFuture<Void> replace(ChangeData cd) throws IOException {
String id = cd.getId().toString();
SolrInputDocument doc = toDocument(cd);
try {
if (cd.getChange().getStatus().isOpen()) {
closedIndex.deleteById(id);
openIndex.add(doc);
} else {
openIndex.deleteById(id);
closedIndex.add(doc);
}
} catch (SolrServerException e) {
throw new IOException(e);
}
commit(openIndex);
commit(closedIndex);
return Futures.immediateFuture(null);
}
@Override
public ListenableFuture<Void> delete(ChangeData cd) throws IOException {
String id = cd.getId().toString();
try {
if (cd.getChange().getStatus().isOpen()) {
openIndex.deleteById(id);
commit(openIndex);
} else {
closedIndex.deleteById(id);
commit(closedIndex);
}
return Futures.immediateFuture(null);
} catch (SolrServerException e) {
throw new IOException(e);
}
}
@Override
public void deleteAll() throws IOException {
try {
openIndex.deleteByQuery("*:*");
closedIndex.deleteByQuery("*:*");
} catch (SolrServerException e) {
throw new IOException(e);
}
commit(openIndex);
commit(closedIndex);
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p)
throws QueryParseException {
Set<Change.Status> statuses = IndexRewriteImpl.getPossibleStatus(p);
List<SolrServer> indexes = Lists.newArrayListWithCapacity(2);
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(openIndex);
}
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(closedIndex);
}
return new QuerySource(indexes, QueryBuilder.toQuery(p));
}
private void commit(SolrServer server) throws IOException {
try {
server.commit();
} catch (SolrServerException e) {
throw new IOException(e);
}
}
private class QuerySource implements ChangeDataSource {
private final List<SolrServer> indexes;
private final SolrQuery query;
public QuerySource(List<SolrServer> indexes, Query q) {
this.indexes = indexes;
query = new SolrQuery(q.toString());
query.setParam("shards.tolerant", true);
query.setFields(ID_FIELD);
query.setSort(
ChangeField.UPDATED.getName(),
SolrQuery.ORDER.desc);
}
@Override
public int getCardinality() {
return 10; // TODO: estimate from solr?
}
@Override
public boolean hasChange() {
return false;
}
@Override
public ResultSet<ChangeData> read() throws OrmException {
try {
// TODO Sort documents during merge to select only top N.
SolrDocumentList docs = new SolrDocumentList();
for (SolrServer index : indexes) {
docs.addAll(index.query(query).getResults());
}
List<ChangeData> result = Lists.newArrayListWithCapacity(docs.size());
for (SolrDocument doc : docs) {
Integer v = (Integer) doc.getFieldValue(ID_FIELD);
result.add(new ChangeData(new Change.Id(v.intValue())));
}
final List<ChangeData> r = Collections.unmodifiableList(result);
return new ResultSet<ChangeData>() {
@Override
public Iterator<ChangeData> iterator() {
return r.iterator();
}
@Override
public List<ChangeData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (SolrServerException e) {
throw new OrmException(e);
}
}
}
private SolrInputDocument toDocument(ChangeData cd) throws IOException {
try {
SolrInputDocument result = new SolrInputDocument();
for (FieldDef<ChangeData, ?> f : ChangeField.ALL.values()) {
if (f.isRepeatable()) {
add(result, f, (Iterable<?>) f.get(cd, fillArgs));
} else {
add(result, f, Collections.singleton(f.get(cd, fillArgs)));
}
}
return result;
} catch (OrmException e) {
throw new IOException(e);
}
}
private void add(SolrInputDocument doc, FieldDef<ChangeData, ?> f,
Iterable<?> values) throws OrmException {
if (f.getType() == FieldType.INTEGER) {
for (Object value : values) {
doc.addField(f.getName(), (Integer) value);
}
} else if (f.getType() == FieldType.LONG) {
for (Object value : values) {
doc.addField(f.getName(), (Long) value);
}
} else if (f.getType() == FieldType.TIMESTAMP) {
for (Object v : values) {
doc.addField(f.getName(), QueryBuilder.toIndexTime((Timestamp) v));
}
} else if (f.getType() == FieldType.EXACT
|| f.getType() == FieldType.PREFIX
|| f.getType() == FieldType.FULL_TEXT) {
for (Object value : values) {
doc.addField(f.getName(), (String) value);
}
} else {
throw QueryBuilder.badFieldType(f.getType());
}
}
@Override
public void finishIndex() throws IOException,
ConfigInvalidException {
// TODO Move the schema version information to a special meta-document
FileBasedConfig cfg = new FileBasedConfig(
solrIndexConfig(sitePaths),
FS.detect());
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
}
cfg.save();
}
}

43
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java Normal file
View File

@ -0,0 +1,43 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.solr;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.IndexModule;
public class SolrIndexModule extends LifecycleModule {
private final boolean checkVersion;
private final int threads;
public SolrIndexModule() {
this(true, 0);
}
public SolrIndexModule(boolean checkVersion, int threads) {
this.checkVersion = checkVersion;
this.threads = threads;
}
@Override
protected void configure() {
install(new IndexModule(threads));
bind(ChangeIndex.class).to(SolrChangeIndex.class);
listener().to(SolrChangeIndex.class);
if (checkVersion) {
listener().to(IndexVersionCheck.class);
}
}
}

1
gerrit-war/BUCK
View File

@ -10,6 +10,7 @@ java_library2(
'//gerrit-reviewdb:server',
'//gerrit-server:common_rules',
'//gerrit-server:server',
'//gerrit-solr:solr',
'//gerrit-sshd:sshd',
'//lib:gwtorm',
'//lib/guice:guice',

16
gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java
View File

@ -50,6 +50,7 @@ import com.google.gerrit.server.schema.DataSourceType;
import com.google.gerrit.server.schema.DatabaseModule;
import com.google.gerrit.server.schema.SchemaModule;
import com.google.gerrit.server.schema.SchemaVersionCheck;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.gerrit.sshd.SshKeyCacheImpl;
import com.google.gerrit.sshd.SshModule;
import com.google.gerrit.sshd.commands.MasterCommandModule;
@ -238,11 +239,18 @@ public class WebAppInitializer extends GuiceServletContextListener {
modules.add(new SmtpEmailSender.Module());
modules.add(new SignedTokenEmailTokenVerifier.Module());
modules.add(new PluginModule());
if (IndexModule.isEnabled(cfgInjector)) {
modules.add(new LuceneIndexModule());
} else {
modules.add(new NoIndexModule());
AbstractModule changeIndexModule;
switch (IndexModule.getIndexType(cfgInjector)) {
case LUCENE:
changeIndexModule = new LuceneIndexModule();
break;
case SOLR:
changeIndexModule = new SolrIndexModule();
break;
default:
changeIndexModule = new NoIndexModule();
}
modules.add(changeIndexModule);
modules.add(new CanonicalWebUrlModule() {
@Override
protected Class<? extends Provider<String>> provider() {

24
lib/BUCK
View File

@ -246,27 +246,3 @@ maven_jar(
visibility = ['//lib:easymock'],
attach_source = False,
)
maven_jar(
name = 'lucene-core',
id = 'org.apache.lucene:lucene-core:4.3.0',
bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3',
src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
],
)
maven_jar(
name = 'lucene-analyzers-common',
id = 'org.apache.lucene:lucene-analyzers-common:4.3.0',
bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56',
src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
],
)

54
lib/commons/BUCK
View File

@ -15,7 +15,10 @@ maven_jar(
license = 'Apache2.0',
exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
attach_source = False,
visibility = ['//lib:velocity'],
visibility = [
'//lib:velocity',
'//lib/solr:zookeeper',
],
)
maven_jar(
@ -39,19 +42,6 @@ maven_jar(
exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
)
maven_jar(
name = 'logging',
id = 'commons-logging:commons-logging:1.1.1',
sha1 = '5043bfebc3db072ed80fbd362e7caf00e885d8ae',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE',
'META-INF/NOTICE',
],
attach_source = False,
visibility = ['//lib/openid:'],
)
maven_jar(
name = 'net',
id = 'commons-net:commons-net:2.2',
@ -77,3 +67,39 @@ maven_jar(
attach_source = False,
exclude = ['META-INF/LICENSE'],
)
maven_jar(
name = 'io',
id = 'commons-io:commons-io:1.4',
sha1 = 'a8762d07e76cfde2395257a5da47ba7c1dbd3dce',
license = 'Apache2.0',
)
maven_jar(
name = 'httpclient',
id = 'org.apache.httpcomponents:httpclient:4.2.5',
bin_sha1 = '666e26e76f2e87d84e4f16acb546481ae1b8e9a6',
src_sha1 = '55d345272944d7e8dace47925336a3764ee0e24b',
license = 'Apache2.0',
deps = [
':codec',
':httpcore',
'//lib/log:jcl-over-slf4j',
],
)
maven_jar(
name = 'httpcore',
id = 'org.apache.httpcomponents:httpcore:4.2.4',
bin_sha1 = '3b7f38df6de5dd8b500e602ae8c2dd5ee446f883',
src_sha1 = 'c3ffe3a73348645042fb0b06303b6a3de194494e',
license = 'Apache2.0',
)
maven_jar(
name = 'httpmime',
id = 'org.apache.httpcomponents:httpmime:4.2.5',
bin_sha1 = '412b9914d0adec6d5716df1ada8acbc4f6f2dd37',
src_sha1 = 'c07ce7f6b153284a9ebaf58532c2442200cf3aa2',
license = 'Apache2.0',
)

7
lib/log/BUCK
View File

@ -22,3 +22,10 @@ maven_jar(
license = 'Apache2.0',
exclude = ['META-INF/LICENSE', 'META-INF/NOTICE'],
)
maven_jar(
name = 'jcl-over-slf4j',
id = 'org.slf4j:jcl-over-slf4j:1.6.1',
sha1 = '99c61095a14dfc9e47a086068033c286bf236475',
license = 'slf4j',
)

49
lib/lucene/BUCK Normal file
View File

@ -0,0 +1,49 @@
include_defs('//lib/maven.defs')
maven_jar(
name = 'core',
id = 'org.apache.lucene:lucene-core:4.3.0',
bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3',
src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
],
)
maven_jar(
name = 'analyzers-common',
id = 'org.apache.lucene:lucene-analyzers-common:4.3.0',
bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56',
src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
],
)
maven_jar(
name = 'highlighter',
id = 'org.apache.lucene:lucene-highlighter:4.3.0',
bin_sha1 = '9e6d60921e16a0d6b2e609c6a02a8b08cd7f643c',
src_sha1 = '0ff70cae1a8fb7af29bf254d90e9885961deed5e',
license = 'Apache2.0',
)
maven_jar(
name = 'queries',
id = 'org.apache.lucene:lucene-queries:4.3.0',
bin_sha1 = '68e01022bdf4f869b95362c9af964846e5d3cf2d',
src_sha1 = '3e3541c1b9f44c532ce88ab6a12216566c3399df',
license = 'Apache2.0',
)
maven_jar(
name = 'spellchecker',
id = 'org.apache.lucene:lucene-spellchecker:3.6.2',
bin_sha1 = '15db0c0cfee44e275f15ad046e46b9a05910ad24',
src_sha1 = 'bbecb3fb725ae594101c165a72c102296007c203',
license = 'Apache2.0',
)

32
lib/openid/BUCK
View File

@ -6,10 +6,10 @@ maven_jar(
sha1 = 'de4f1b33d3b0f0b2ab1d32834ec1190b39db4160',
license = 'Apache2.0',
deps = [
':httpclient',
':nekohtml',
':xerces',
'//lib/commons:logging',
'//lib/commons:httpclient',
'//lib/log:jcl-over-slf4j',
'//lib/guice:guice',
],
visibility = ['PUBLIC'],
@ -33,31 +33,3 @@ maven_jar(
attach_source = False,
visibility = [],
)
maven_jar(
name = 'httpclient',
id = 'org.apache.httpcomponents:httpclient:4.1',
sha1 = '93cd011acb220de08b57d96106e5800d7097742b',
license = 'Apache2.0',
deps = [
':httpcore',
'//lib/commons:codec',
'//lib/commons:logging',
],
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
],
visibility = ['//gerrit-acceptance-tests:'],
)
maven_jar(
name = 'httpcore',
id = 'org.apache.httpcomponents:httpcore:4.1',
sha1 = '33fc26c02f8043ab0ede19eadc8c9885386b255c',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
],
)

33
lib/solr/BUCK Normal file
View File

@ -0,0 +1,33 @@
include_defs('//lib/maven.defs')
# Java client library to use Solr over the network.
maven_jar(
name = 'solrj',
id = 'org.apache.solr:solr-solrj:4.3.1',
sha1 = '433fe37796e67eaeb4452f69eb1fae2de27cb7a8',
license = 'Apache2.0',
deps = [
':noggit',
':zookeeper',
'//lib/commons:httpclient',
'//lib/commons:httpmime',
'//lib/commons:io',
],
)
maven_jar(
name = 'noggit',
id = 'org.noggit:noggit:0.5',
sha1 = '8e6e65624d2e09a30190c6434abe23b7d4e5413c',
license = 'Apache2.0',
visibility = [],
)
maven_jar(
name = 'zookeeper',
id = 'org.apache.zookeeper:zookeeper:3.4.5',
sha1 = 'c0f69fb36526552a8f0bc548a6c33c49cf08e562',
license = 'Apache2.0',
deps = ['//lib/log:api'],
visibility = [],
)