Add secondary index implementation using SolrCloud
SolrCloud can be used instead of Lucene by adding "type = SOLR" under [index] and "url = <zookeeper-url>" under [index "solr"] in gerrit.config. Change-Id: I0ff8579c5e23c58b16f3605bc20eba4e80fb40fc
This commit is contained in:
parent
9279b29da7
commit
404c8246bc
@ -24,13 +24,13 @@ java_test(
|
|||||||
'//lib:junit',
|
'//lib:junit',
|
||||||
'//lib:servlet-api-3_0',
|
'//lib:servlet-api-3_0',
|
||||||
|
|
||||||
|
'//lib/commons:httpclient',
|
||||||
|
'//lib/commons:httpcore',
|
||||||
'//lib/log:impl_log4j',
|
'//lib/log:impl_log4j',
|
||||||
'//lib/log:log4j',
|
'//lib/log:log4j',
|
||||||
'//lib/guice:guice',
|
'//lib/guice:guice',
|
||||||
'//lib/jgit:jgit',
|
'//lib/jgit:jgit',
|
||||||
'//lib/jgit:junit',
|
'//lib/jgit:junit',
|
||||||
'//lib/openid:httpclient',
|
|
||||||
'//lib/openid:httpcore',
|
|
||||||
],
|
],
|
||||||
source_under_test = TEST,
|
source_under_test = TEST,
|
||||||
labels = ['slow'],
|
labels = ['slow'],
|
||||||
|
@ -1,18 +1,36 @@
|
|||||||
|
QUERY_BUILDER = [
|
||||||
|
'src/main/java/com/google/gerrit/lucene/QueryBuilder.java',
|
||||||
|
]
|
||||||
|
|
||||||
|
java_library(
|
||||||
|
name = 'query_builder',
|
||||||
|
srcs = QUERY_BUILDER,
|
||||||
|
deps = [
|
||||||
|
'//gerrit-antlr:query_exception',
|
||||||
|
'//gerrit-reviewdb:server',
|
||||||
|
'//gerrit-server:server',
|
||||||
|
'//lib:gwtorm',
|
||||||
|
'//lib/lucene:core',
|
||||||
|
],
|
||||||
|
visibility = ['PUBLIC'],
|
||||||
|
)
|
||||||
|
|
||||||
java_library(
|
java_library(
|
||||||
name = 'lucene',
|
name = 'lucene',
|
||||||
srcs = glob(['src/main/java/**/*.java']),
|
srcs = glob(['src/main/java/**/*.java'], excludes = QUERY_BUILDER),
|
||||||
deps = [
|
deps = [
|
||||||
|
':query_builder',
|
||||||
'//gerrit-antlr:query_exception',
|
'//gerrit-antlr:query_exception',
|
||||||
'//gerrit-extension-api:api',
|
'//gerrit-extension-api:api',
|
||||||
'//gerrit-reviewdb:server',
|
'//gerrit-reviewdb:server',
|
||||||
'//gerrit-server:server',
|
'//gerrit-server:server',
|
||||||
'//lib:guava',
|
'//lib:guava',
|
||||||
'//lib:gwtorm',
|
'//lib:gwtorm',
|
||||||
'//lib:lucene-analyzers-common',
|
|
||||||
'//lib:lucene-core',
|
|
||||||
'//lib/guice:guice',
|
'//lib/guice:guice',
|
||||||
'//lib/jgit:jgit',
|
'//lib/jgit:jgit',
|
||||||
'//lib/log:api',
|
'//lib/log:api',
|
||||||
|
'//lib/lucene:analyzers-common',
|
||||||
|
'//lib/lucene:core',
|
||||||
],
|
],
|
||||||
visibility = ['PUBLIC'],
|
visibility = ['PUBLIC'],
|
||||||
)
|
)
|
||||||
|
@ -14,11 +14,10 @@
|
|||||||
|
|
||||||
package com.google.gerrit.lucene;
|
package com.google.gerrit.lucene;
|
||||||
|
|
||||||
|
import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
|
||||||
|
import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
|
||||||
import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES;
|
import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES;
|
||||||
import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES;
|
import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES;
|
||||||
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
|
|
||||||
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
|
|
||||||
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
|
|
||||||
|
|
||||||
import com.google.common.base.Function;
|
import com.google.common.base.Function;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
@ -35,18 +34,11 @@ import com.google.gerrit.server.index.ChangeIndex;
|
|||||||
import com.google.gerrit.server.index.FieldDef;
|
import com.google.gerrit.server.index.FieldDef;
|
||||||
import com.google.gerrit.server.index.FieldDef.FillArgs;
|
import com.google.gerrit.server.index.FieldDef.FillArgs;
|
||||||
import com.google.gerrit.server.index.FieldType;
|
import com.google.gerrit.server.index.FieldType;
|
||||||
import com.google.gerrit.server.index.IndexPredicate;
|
|
||||||
import com.google.gerrit.server.index.RegexPredicate;
|
|
||||||
import com.google.gerrit.server.index.TimestampRangePredicate;
|
|
||||||
import com.google.gerrit.server.query.AndPredicate;
|
|
||||||
import com.google.gerrit.server.query.NotPredicate;
|
|
||||||
import com.google.gerrit.server.query.OrPredicate;
|
|
||||||
import com.google.gerrit.server.query.Predicate;
|
import com.google.gerrit.server.query.Predicate;
|
||||||
import com.google.gerrit.server.query.QueryParseException;
|
import com.google.gerrit.server.query.QueryParseException;
|
||||||
import com.google.gerrit.server.query.change.ChangeData;
|
import com.google.gerrit.server.query.change.ChangeData;
|
||||||
import com.google.gerrit.server.query.change.ChangeDataSource;
|
import com.google.gerrit.server.query.change.ChangeDataSource;
|
||||||
import com.google.gerrit.server.query.change.IndexRewriteImpl;
|
import com.google.gerrit.server.query.change.IndexRewriteImpl;
|
||||||
import com.google.gerrit.server.query.change.SortKeyPredicate;
|
|
||||||
import com.google.gwtorm.server.OrmException;
|
import com.google.gwtorm.server.OrmException;
|
||||||
import com.google.gwtorm.server.ResultSet;
|
import com.google.gwtorm.server.ResultSet;
|
||||||
|
|
||||||
@ -62,24 +54,18 @@ import org.apache.lucene.index.IndexWriter;
|
|||||||
import org.apache.lucene.index.IndexWriterConfig;
|
import org.apache.lucene.index.IndexWriterConfig;
|
||||||
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
||||||
import org.apache.lucene.index.Term;
|
import org.apache.lucene.index.Term;
|
||||||
import org.apache.lucene.search.BooleanClause;
|
|
||||||
import org.apache.lucene.search.BooleanQuery;
|
|
||||||
import org.apache.lucene.search.IndexSearcher;
|
import org.apache.lucene.search.IndexSearcher;
|
||||||
import org.apache.lucene.search.NumericRangeQuery;
|
|
||||||
import org.apache.lucene.search.PrefixQuery;
|
|
||||||
import org.apache.lucene.search.FuzzyQuery;
|
|
||||||
import org.apache.lucene.search.Query;
|
import org.apache.lucene.search.Query;
|
||||||
import org.apache.lucene.search.RegexpQuery;
|
|
||||||
import org.apache.lucene.search.ScoreDoc;
|
import org.apache.lucene.search.ScoreDoc;
|
||||||
import org.apache.lucene.search.SearcherManager;
|
import org.apache.lucene.search.SearcherManager;
|
||||||
import org.apache.lucene.search.Sort;
|
import org.apache.lucene.search.Sort;
|
||||||
import org.apache.lucene.search.SortField;
|
import org.apache.lucene.search.SortField;
|
||||||
import org.apache.lucene.search.TermQuery;
|
|
||||||
import org.apache.lucene.search.TopDocs;
|
import org.apache.lucene.search.TopDocs;
|
||||||
import org.apache.lucene.util.BytesRef;
|
|
||||||
import org.apache.lucene.util.NumericUtils;
|
|
||||||
import org.apache.lucene.util.Version;
|
import org.apache.lucene.util.Version;
|
||||||
|
import org.eclipse.jgit.errors.ConfigInvalidException;
|
||||||
import org.eclipse.jgit.lib.Config;
|
import org.eclipse.jgit.lib.Config;
|
||||||
|
import org.eclipse.jgit.storage.file.FileBasedConfig;
|
||||||
|
import org.eclipse.jgit.util.FS;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
@ -89,6 +75,7 @@ import java.sql.Timestamp;
|
|||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.Future;
|
import java.util.concurrent.Future;
|
||||||
@ -122,6 +109,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
return writerConfig;
|
return writerConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private final SitePaths sitePaths;
|
||||||
private final FillArgs fillArgs;
|
private final FillArgs fillArgs;
|
||||||
private final ExecutorService executor;
|
private final ExecutorService executor;
|
||||||
private final boolean readOnly;
|
private final boolean readOnly;
|
||||||
@ -131,6 +119,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
LuceneChangeIndex(Config cfg, SitePaths sitePaths,
|
LuceneChangeIndex(Config cfg, SitePaths sitePaths,
|
||||||
ListeningScheduledExecutorService executor, FillArgs fillArgs,
|
ListeningScheduledExecutorService executor, FillArgs fillArgs,
|
||||||
boolean readOnly) throws IOException {
|
boolean readOnly) throws IOException {
|
||||||
|
this.sitePaths = sitePaths;
|
||||||
this.fillArgs = fillArgs;
|
this.fillArgs = fillArgs;
|
||||||
this.executor = executor;
|
this.executor = executor;
|
||||||
this.readOnly = readOnly;
|
this.readOnly = readOnly;
|
||||||
@ -167,7 +156,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@Override
|
@Override
|
||||||
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
|
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
|
||||||
Term id = idTerm(cd);
|
Term id = QueryBuilder.idTerm(cd);
|
||||||
Document doc = toDocument(cd);
|
Document doc = toDocument(cd);
|
||||||
if (readOnly) {
|
if (readOnly) {
|
||||||
return Futures.immediateFuture(null);
|
return Futures.immediateFuture(null);
|
||||||
@ -187,7 +176,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@Override
|
@Override
|
||||||
public ListenableFuture<Void> replace(ChangeData cd) throws IOException {
|
public ListenableFuture<Void> replace(ChangeData cd) throws IOException {
|
||||||
Term id = idTerm(cd);
|
Term id = QueryBuilder.idTerm(cd);
|
||||||
Document doc = toDocument(cd);
|
Document doc = toDocument(cd);
|
||||||
if (readOnly) {
|
if (readOnly) {
|
||||||
return Futures.immediateFuture(null);
|
return Futures.immediateFuture(null);
|
||||||
@ -206,7 +195,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@Override
|
@Override
|
||||||
public ListenableFuture<Void> delete(ChangeData cd) throws IOException {
|
public ListenableFuture<Void> delete(ChangeData cd) throws IOException {
|
||||||
Term id = idTerm(cd);
|
Term id = QueryBuilder.idTerm(cd);
|
||||||
if (readOnly) {
|
if (readOnly) {
|
||||||
return Futures.immediateFuture(null);
|
return Futures.immediateFuture(null);
|
||||||
}
|
}
|
||||||
@ -226,6 +215,11 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteAll() throws IOException {
|
||||||
|
openIndex.deleteAll();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
||||||
throws QueryParseException {
|
throws QueryParseException {
|
||||||
@ -237,138 +231,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
|
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
|
||||||
indexes.add(closedIndex);
|
indexes.add(closedIndex);
|
||||||
}
|
}
|
||||||
return new QuerySource(indexes, toQuery(p));
|
return new QuerySource(indexes, QueryBuilder.toQuery(p));
|
||||||
}
|
|
||||||
|
|
||||||
private Term idTerm(ChangeData cd) {
|
|
||||||
return intTerm(ID_FIELD, cd.getId().get());
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query toQuery(Predicate<ChangeData> p) throws QueryParseException {
|
|
||||||
if (p.getClass() == AndPredicate.class) {
|
|
||||||
return booleanQuery(p, MUST);
|
|
||||||
} else if (p.getClass() == OrPredicate.class) {
|
|
||||||
return booleanQuery(p, SHOULD);
|
|
||||||
} else if (p.getClass() == NotPredicate.class) {
|
|
||||||
if (p.getChild(0) instanceof TimestampRangePredicate) {
|
|
||||||
return notTimestampQuery(
|
|
||||||
(TimestampRangePredicate<ChangeData>) p.getChild(0));
|
|
||||||
}
|
|
||||||
return booleanQuery(p, MUST_NOT);
|
|
||||||
} else if (p instanceof IndexPredicate) {
|
|
||||||
return fieldQuery((IndexPredicate<ChangeData>) p);
|
|
||||||
} else {
|
|
||||||
throw new QueryParseException("Cannot convert to index predicate: " + p);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query booleanQuery(Predicate<ChangeData> p, BooleanClause.Occur o)
|
|
||||||
throws QueryParseException {
|
|
||||||
BooleanQuery q = new BooleanQuery();
|
|
||||||
for (int i = 0; i < p.getChildCount(); i++) {
|
|
||||||
q.add(toQuery(p.getChild(i)), o);
|
|
||||||
}
|
|
||||||
return q;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query fieldQuery(IndexPredicate<ChangeData> p)
|
|
||||||
throws QueryParseException {
|
|
||||||
if (p.getType() == FieldType.INTEGER) {
|
|
||||||
return intQuery(p);
|
|
||||||
} else if (p.getType() == FieldType.TIMESTAMP) {
|
|
||||||
return timestampQuery(p);
|
|
||||||
} else if (p.getType() == FieldType.EXACT) {
|
|
||||||
return exactQuery(p);
|
|
||||||
} else if (p.getType() == FieldType.PREFIX) {
|
|
||||||
return prefixQuery(p);
|
|
||||||
} else if (p.getType() == FieldType.FULL_TEXT) {
|
|
||||||
return fullTextQuery(p);
|
|
||||||
} else if (p instanceof SortKeyPredicate) {
|
|
||||||
return sortKeyQuery((SortKeyPredicate) p);
|
|
||||||
} else {
|
|
||||||
throw badFieldType(p.getType());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Term intTerm(String name, int value) {
|
|
||||||
BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
|
|
||||||
NumericUtils.intToPrefixCodedBytes(value, 0, bytes);
|
|
||||||
return new Term(name, bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query intQuery(IndexPredicate<ChangeData> p)
|
|
||||||
throws QueryParseException {
|
|
||||||
int value;
|
|
||||||
try {
|
|
||||||
// Can't use IntPredicate because it and IndexPredicate are different
|
|
||||||
// subclasses of OperatorPredicate.
|
|
||||||
value = Integer.valueOf(p.getValue());
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
throw new QueryParseException("not an integer: " + p.getValue());
|
|
||||||
}
|
|
||||||
return new TermQuery(intTerm(p.getField().getName(), value));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Query sortKeyQuery(SortKeyPredicate p) {
|
|
||||||
return NumericRangeQuery.newLongRange(
|
|
||||||
p.getField().getName(),
|
|
||||||
p.getMinValue(),
|
|
||||||
p.getMaxValue(),
|
|
||||||
true, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Query timestampQuery(IndexPredicate<ChangeData> p)
|
|
||||||
throws QueryParseException {
|
|
||||||
if (p instanceof TimestampRangePredicate) {
|
|
||||||
TimestampRangePredicate<ChangeData> r =
|
|
||||||
(TimestampRangePredicate<ChangeData>) p;
|
|
||||||
return NumericRangeQuery.newIntRange(
|
|
||||||
r.getField().getName(),
|
|
||||||
toIndexTime(r.getMinTimestamp()),
|
|
||||||
toIndexTime(r.getMaxTimestamp()),
|
|
||||||
true, true);
|
|
||||||
}
|
|
||||||
throw new QueryParseException("not a timestamp: " + p);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Query notTimestampQuery(TimestampRangePredicate<ChangeData> r)
|
|
||||||
throws QueryParseException {
|
|
||||||
if (r.getMinTimestamp().getTime() == 0) {
|
|
||||||
return NumericRangeQuery.newIntRange(
|
|
||||||
r.getField().getName(),
|
|
||||||
toIndexTime(r.getMaxTimestamp()),
|
|
||||||
null,
|
|
||||||
true, true);
|
|
||||||
}
|
|
||||||
throw new QueryParseException("cannot negate: " + r);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query exactQuery(IndexPredicate<ChangeData> p) {
|
|
||||||
if (p instanceof RegexPredicate<?>) {
|
|
||||||
return regexQuery(p);
|
|
||||||
} else {
|
|
||||||
return new TermQuery(new Term(p.getField().getName(), p.getValue()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query regexQuery(IndexPredicate<ChangeData> p) {
|
|
||||||
String re = p.getValue();
|
|
||||||
if (re.startsWith("^")) {
|
|
||||||
re = re.substring(1);
|
|
||||||
}
|
|
||||||
if (re.endsWith("$") && !re.endsWith("\\$")) {
|
|
||||||
re = re.substring(0, re.length() - 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
return new RegexpQuery(new Term(p.getField().getName(), re));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query prefixQuery(IndexPredicate<ChangeData> p) {
|
|
||||||
return new PrefixQuery(new Term(p.getField().getName(), p.getValue()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Query fullTextQuery(IndexPredicate<ChangeData> p) {
|
|
||||||
return new FuzzyQuery(new Term(p.getField().getName(), p.getValue()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class QuerySource implements ChangeDataSource {
|
private static class QuerySource implements ChangeDataSource {
|
||||||
@ -485,7 +348,8 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
}
|
}
|
||||||
} else if (f.getType() == FieldType.TIMESTAMP) {
|
} else if (f.getType() == FieldType.TIMESTAMP) {
|
||||||
for (Object v : values) {
|
for (Object v : values) {
|
||||||
doc.add(new IntField(name, toIndexTime((Timestamp) v), store));
|
int t = QueryBuilder.toIndexTime((Timestamp) v);
|
||||||
|
doc.add(new IntField(name, t, store));
|
||||||
}
|
}
|
||||||
} else if (f.getType() == FieldType.EXACT
|
} else if (f.getType() == FieldType.EXACT
|
||||||
|| f.getType() == FieldType.PREFIX) {
|
|| f.getType() == FieldType.PREFIX) {
|
||||||
@ -497,19 +361,24 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
|
|||||||
doc.add(new TextField(name, (String) value, store));
|
doc.add(new TextField(name, (String) value, store));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
throw badFieldType(f.getType());
|
throw QueryBuilder.badFieldType(f.getType());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int toIndexTime(Timestamp ts) {
|
|
||||||
return (int) (ts.getTime() / 60000);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Field.Store store(FieldDef<?, ?> f) {
|
private static Field.Store store(FieldDef<?, ?> f) {
|
||||||
return f.isStored() ? Field.Store.YES : Field.Store.NO;
|
return f.isStored() ? Field.Store.YES : Field.Store.NO;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IllegalArgumentException badFieldType(FieldType<?> t) {
|
@Override
|
||||||
return new IllegalArgumentException("unknown index field type " + t);
|
public void finishIndex() throws IOException,
|
||||||
|
ConfigInvalidException {
|
||||||
|
FileBasedConfig cfg =
|
||||||
|
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
|
||||||
|
|
||||||
|
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
|
||||||
|
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
|
||||||
|
}
|
||||||
|
cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
|
||||||
|
cfg.save();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,192 @@
|
|||||||
|
// Copyright (C) 2013 The Android Open Source Project
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package com.google.gerrit.lucene;
|
||||||
|
|
||||||
|
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
|
||||||
|
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
|
||||||
|
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
|
||||||
|
|
||||||
|
import com.google.gerrit.server.index.ChangeField;
|
||||||
|
import com.google.gerrit.server.index.FieldType;
|
||||||
|
import com.google.gerrit.server.index.IndexPredicate;
|
||||||
|
import com.google.gerrit.server.index.RegexPredicate;
|
||||||
|
import com.google.gerrit.server.index.TimestampRangePredicate;
|
||||||
|
import com.google.gerrit.server.query.AndPredicate;
|
||||||
|
import com.google.gerrit.server.query.NotPredicate;
|
||||||
|
import com.google.gerrit.server.query.OrPredicate;
|
||||||
|
import com.google.gerrit.server.query.Predicate;
|
||||||
|
import com.google.gerrit.server.query.QueryParseException;
|
||||||
|
import com.google.gerrit.server.query.change.ChangeData;
|
||||||
|
import com.google.gerrit.server.query.change.SortKeyPredicate;
|
||||||
|
|
||||||
|
import org.apache.lucene.index.Term;
|
||||||
|
import org.apache.lucene.search.BooleanClause;
|
||||||
|
import org.apache.lucene.search.BooleanQuery;
|
||||||
|
import org.apache.lucene.search.FuzzyQuery;
|
||||||
|
import org.apache.lucene.search.NumericRangeQuery;
|
||||||
|
import org.apache.lucene.search.PrefixQuery;
|
||||||
|
import org.apache.lucene.search.Query;
|
||||||
|
import org.apache.lucene.search.RegexpQuery;
|
||||||
|
import org.apache.lucene.search.TermQuery;
|
||||||
|
import org.apache.lucene.util.BytesRef;
|
||||||
|
import org.apache.lucene.util.NumericUtils;
|
||||||
|
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
|
||||||
|
public class QueryBuilder {
|
||||||
|
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
|
||||||
|
|
||||||
|
public static Term idTerm(ChangeData cd) {
|
||||||
|
return intTerm(ID_FIELD, cd.getId().get());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Query toQuery(Predicate<ChangeData> p)
|
||||||
|
throws QueryParseException {
|
||||||
|
if (p.getClass() == AndPredicate.class) {
|
||||||
|
return booleanQuery(p, MUST);
|
||||||
|
} else if (p.getClass() == OrPredicate.class) {
|
||||||
|
return booleanQuery(p, SHOULD);
|
||||||
|
} else if (p.getClass() == NotPredicate.class) {
|
||||||
|
if (p.getChild(0) instanceof TimestampRangePredicate) {
|
||||||
|
return notTimestampQuery(
|
||||||
|
(TimestampRangePredicate<ChangeData>) p.getChild(0));
|
||||||
|
}
|
||||||
|
return booleanQuery(p, MUST_NOT);
|
||||||
|
} else if (p instanceof IndexPredicate) {
|
||||||
|
return fieldQuery((IndexPredicate<ChangeData>) p);
|
||||||
|
} else {
|
||||||
|
throw new QueryParseException("Cannot convert to index predicate: " + p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query booleanQuery(Predicate<ChangeData> p, BooleanClause.Occur o)
|
||||||
|
throws QueryParseException {
|
||||||
|
BooleanQuery q = new BooleanQuery();
|
||||||
|
for (int i = 0; i < p.getChildCount(); i++) {
|
||||||
|
q.add(toQuery(p.getChild(i)), o);
|
||||||
|
}
|
||||||
|
return q;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query fieldQuery(IndexPredicate<ChangeData> p)
|
||||||
|
throws QueryParseException {
|
||||||
|
if (p.getType() == FieldType.INTEGER) {
|
||||||
|
return intQuery(p);
|
||||||
|
} else if (p.getType() == FieldType.TIMESTAMP) {
|
||||||
|
return timestampQuery(p);
|
||||||
|
} else if (p.getType() == FieldType.EXACT) {
|
||||||
|
return exactQuery(p);
|
||||||
|
} else if (p.getType() == FieldType.PREFIX) {
|
||||||
|
return prefixQuery(p);
|
||||||
|
} else if (p.getType() == FieldType.FULL_TEXT) {
|
||||||
|
return fullTextQuery(p);
|
||||||
|
} else if (p instanceof SortKeyPredicate) {
|
||||||
|
return sortKeyQuery((SortKeyPredicate) p);
|
||||||
|
} else {
|
||||||
|
throw badFieldType(p.getType());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Term intTerm(String name, int value) {
|
||||||
|
BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
|
||||||
|
NumericUtils.intToPrefixCodedBytes(value, 0, bytes);
|
||||||
|
return new Term(name, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query intQuery(IndexPredicate<ChangeData> p)
|
||||||
|
throws QueryParseException {
|
||||||
|
int value;
|
||||||
|
try {
|
||||||
|
// Can't use IntPredicate because it and IndexPredicate are different
|
||||||
|
// subclasses of OperatorPredicate.
|
||||||
|
value = Integer.valueOf(p.getValue());
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
throw new QueryParseException("not an integer: " + p.getValue());
|
||||||
|
}
|
||||||
|
return new TermQuery(intTerm(p.getField().getName(), value));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query sortKeyQuery(SortKeyPredicate p) {
|
||||||
|
return NumericRangeQuery.newLongRange(
|
||||||
|
p.getField().getName(),
|
||||||
|
p.getMinValue(),
|
||||||
|
p.getMaxValue(),
|
||||||
|
true, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query timestampQuery(IndexPredicate<ChangeData> p)
|
||||||
|
throws QueryParseException {
|
||||||
|
if (p instanceof TimestampRangePredicate) {
|
||||||
|
TimestampRangePredicate<ChangeData> r =
|
||||||
|
(TimestampRangePredicate<ChangeData>) p;
|
||||||
|
return NumericRangeQuery.newIntRange(
|
||||||
|
r.getField().getName(),
|
||||||
|
toIndexTime(r.getMinTimestamp()),
|
||||||
|
toIndexTime(r.getMaxTimestamp()),
|
||||||
|
true, true);
|
||||||
|
}
|
||||||
|
throw new QueryParseException("not a timestamp: " + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query notTimestampQuery(TimestampRangePredicate<ChangeData> r)
|
||||||
|
throws QueryParseException {
|
||||||
|
if (r.getMinTimestamp().getTime() == 0) {
|
||||||
|
return NumericRangeQuery.newIntRange(
|
||||||
|
r.getField().getName(),
|
||||||
|
toIndexTime(r.getMaxTimestamp()),
|
||||||
|
null,
|
||||||
|
true, true);
|
||||||
|
}
|
||||||
|
throw new QueryParseException("cannot negate: " + r);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query exactQuery(IndexPredicate<ChangeData> p) {
|
||||||
|
if (p instanceof RegexPredicate<?>) {
|
||||||
|
return regexQuery(p);
|
||||||
|
} else {
|
||||||
|
return new TermQuery(new Term(p.getField().getName(), p.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query regexQuery(IndexPredicate<ChangeData> p) {
|
||||||
|
String re = p.getValue();
|
||||||
|
if (re.startsWith("^")) {
|
||||||
|
re = re.substring(1);
|
||||||
|
}
|
||||||
|
if (re.endsWith("$") && !re.endsWith("\\$")) {
|
||||||
|
re = re.substring(0, re.length() - 1);
|
||||||
|
}
|
||||||
|
return new RegexpQuery(new Term(p.getField().getName(), re));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query prefixQuery(IndexPredicate<ChangeData> p) {
|
||||||
|
return new PrefixQuery(new Term(p.getField().getName(), p.getValue()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Query fullTextQuery(IndexPredicate<ChangeData> p) {
|
||||||
|
return new FuzzyQuery(new Term(p.getField().getName(), p.getValue()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int toIndexTime(Timestamp ts) {
|
||||||
|
return (int) (ts.getTime() / 60000);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IllegalArgumentException badFieldType(FieldType<?> t) {
|
||||||
|
return new IllegalArgumentException("unknown index field type " + t);
|
||||||
|
}
|
||||||
|
|
||||||
|
private QueryBuilder() {
|
||||||
|
}
|
||||||
|
}
|
@ -114,6 +114,10 @@ class SubIndex {
|
|||||||
return new NrtFuture(writer.deleteDocuments(term));
|
return new NrtFuture(writer.deleteDocuments(term));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteAll() throws IOException {
|
||||||
|
writer.deleteAll();
|
||||||
|
}
|
||||||
|
|
||||||
IndexSearcher acquire() throws IOException {
|
IndexSearcher acquire() throws IOException {
|
||||||
return nrtManager.acquire();
|
return nrtManager.acquire();
|
||||||
}
|
}
|
||||||
|
@ -13,6 +13,7 @@ java_library2(
|
|||||||
'//gerrit-server:common_rules',
|
'//gerrit-server:common_rules',
|
||||||
'//gerrit-reviewdb:server',
|
'//gerrit-reviewdb:server',
|
||||||
'//gerrit-server:server',
|
'//gerrit-server:server',
|
||||||
|
'//gerrit-solr:solr',
|
||||||
'//gerrit-sshd:sshd',
|
'//gerrit-sshd:sshd',
|
||||||
'//gerrit-util-cli:cli',
|
'//gerrit-util-cli:cli',
|
||||||
'//lib:args4j',
|
'//lib:args4j',
|
||||||
@ -31,7 +32,7 @@ java_library2(
|
|||||||
'//lib/jgit:jgit',
|
'//lib/jgit:jgit',
|
||||||
'//lib/log:api',
|
'//lib/log:api',
|
||||||
'//lib/log:log4j',
|
'//lib/log:log4j',
|
||||||
'//lib:lucene-core',
|
'//lib/lucene:core',
|
||||||
'//lib/mina:sshd',
|
'//lib/mina:sshd',
|
||||||
'//lib/prolog:prolog-cafe',
|
'//lib/prolog:prolog-cafe',
|
||||||
],
|
],
|
||||||
|
@ -59,6 +59,7 @@ import com.google.gerrit.server.plugins.PluginModule;
|
|||||||
import com.google.gerrit.server.schema.SchemaVersionCheck;
|
import com.google.gerrit.server.schema.SchemaVersionCheck;
|
||||||
import com.google.gerrit.server.ssh.NoSshKeyCache;
|
import com.google.gerrit.server.ssh.NoSshKeyCache;
|
||||||
import com.google.gerrit.server.ssh.NoSshModule;
|
import com.google.gerrit.server.ssh.NoSshModule;
|
||||||
|
import com.google.gerrit.solr.SolrIndexModule;
|
||||||
import com.google.gerrit.sshd.SshKeyCacheImpl;
|
import com.google.gerrit.sshd.SshKeyCacheImpl;
|
||||||
import com.google.gerrit.sshd.SshModule;
|
import com.google.gerrit.sshd.SshModule;
|
||||||
import com.google.gerrit.sshd.commands.MasterCommandModule;
|
import com.google.gerrit.sshd.commands.MasterCommandModule;
|
||||||
@ -253,11 +254,18 @@ public class Daemon extends SiteProgram {
|
|||||||
modules.add(new SmtpEmailSender.Module());
|
modules.add(new SmtpEmailSender.Module());
|
||||||
modules.add(new SignedTokenEmailTokenVerifier.Module());
|
modules.add(new SignedTokenEmailTokenVerifier.Module());
|
||||||
modules.add(new PluginModule());
|
modules.add(new PluginModule());
|
||||||
if (IndexModule.isEnabled(cfgInjector)) {
|
AbstractModule changeIndexModule;
|
||||||
modules.add(new LuceneIndexModule());
|
switch (IndexModule.getIndexType(cfgInjector)) {
|
||||||
} else {
|
case LUCENE:
|
||||||
modules.add(new NoIndexModule());
|
changeIndexModule = new LuceneIndexModule();
|
||||||
|
break;
|
||||||
|
case SOLR:
|
||||||
|
changeIndexModule = new SolrIndexModule();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
changeIndexModule = new NoIndexModule();
|
||||||
}
|
}
|
||||||
|
modules.add(changeIndexModule);
|
||||||
if (httpd) {
|
if (httpd) {
|
||||||
modules.add(new CanonicalWebUrlModule() {
|
modules.add(new CanonicalWebUrlModule() {
|
||||||
@Override
|
@Override
|
||||||
|
@ -14,9 +14,6 @@
|
|||||||
|
|
||||||
package com.google.gerrit.pgm;
|
package com.google.gerrit.pgm;
|
||||||
|
|
||||||
import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
|
|
||||||
import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
|
|
||||||
import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
|
|
||||||
import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER;
|
import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER;
|
||||||
|
|
||||||
import com.google.common.base.Stopwatch;
|
import com.google.common.base.Stopwatch;
|
||||||
@ -41,16 +38,19 @@ import com.google.gerrit.reviewdb.client.Project;
|
|||||||
import com.google.gerrit.reviewdb.server.ReviewDb;
|
import com.google.gerrit.reviewdb.server.ReviewDb;
|
||||||
import com.google.gerrit.server.cache.CacheRemovalListener;
|
import com.google.gerrit.server.cache.CacheRemovalListener;
|
||||||
import com.google.gerrit.server.cache.h2.DefaultCacheFactory;
|
import com.google.gerrit.server.cache.h2.DefaultCacheFactory;
|
||||||
import com.google.gerrit.server.config.SitePaths;
|
|
||||||
import com.google.gerrit.server.git.GitRepositoryManager;
|
import com.google.gerrit.server.git.GitRepositoryManager;
|
||||||
import com.google.gerrit.server.git.MultiProgressMonitor;
|
import com.google.gerrit.server.git.MultiProgressMonitor;
|
||||||
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
|
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
|
||||||
|
import com.google.gerrit.server.index.ChangeIndex;
|
||||||
import com.google.gerrit.server.index.ChangeIndexer;
|
import com.google.gerrit.server.index.ChangeIndexer;
|
||||||
import com.google.gerrit.server.index.IndexExecutor;
|
import com.google.gerrit.server.index.IndexExecutor;
|
||||||
import com.google.gerrit.server.index.IndexModule;
|
import com.google.gerrit.server.index.IndexModule;
|
||||||
|
import com.google.gerrit.server.index.IndexModule.IndexType;
|
||||||
|
import com.google.gerrit.server.index.NoIndexModule;
|
||||||
import com.google.gerrit.server.patch.PatchListCacheImpl;
|
import com.google.gerrit.server.patch.PatchListCacheImpl;
|
||||||
import com.google.gerrit.server.patch.PatchListLoader;
|
import com.google.gerrit.server.patch.PatchListLoader;
|
||||||
import com.google.gerrit.server.query.change.ChangeData;
|
import com.google.gerrit.server.query.change.ChangeData;
|
||||||
|
import com.google.gerrit.solr.SolrIndexModule;
|
||||||
import com.google.gwtorm.server.OrmException;
|
import com.google.gwtorm.server.OrmException;
|
||||||
import com.google.gwtorm.server.SchemaFactory;
|
import com.google.gwtorm.server.SchemaFactory;
|
||||||
import com.google.inject.AbstractModule;
|
import com.google.inject.AbstractModule;
|
||||||
@ -61,8 +61,6 @@ import com.google.inject.Provider;
|
|||||||
import com.google.inject.ProvisionException;
|
import com.google.inject.ProvisionException;
|
||||||
import com.google.inject.TypeLiteral;
|
import com.google.inject.TypeLiteral;
|
||||||
|
|
||||||
import org.apache.lucene.store.Directory;
|
|
||||||
import org.apache.lucene.store.FSDirectory;
|
|
||||||
import org.eclipse.jgit.diff.DiffEntry;
|
import org.eclipse.jgit.diff.DiffEntry;
|
||||||
import org.eclipse.jgit.diff.DiffFormatter;
|
import org.eclipse.jgit.diff.DiffFormatter;
|
||||||
import org.eclipse.jgit.errors.ConfigInvalidException;
|
import org.eclipse.jgit.errors.ConfigInvalidException;
|
||||||
@ -78,14 +76,11 @@ import org.eclipse.jgit.revwalk.RevCommit;
|
|||||||
import org.eclipse.jgit.revwalk.RevObject;
|
import org.eclipse.jgit.revwalk.RevObject;
|
||||||
import org.eclipse.jgit.revwalk.RevTree;
|
import org.eclipse.jgit.revwalk.RevTree;
|
||||||
import org.eclipse.jgit.revwalk.RevWalk;
|
import org.eclipse.jgit.revwalk.RevWalk;
|
||||||
import org.eclipse.jgit.storage.file.FileBasedConfig;
|
|
||||||
import org.eclipse.jgit.util.FS;
|
|
||||||
import org.eclipse.jgit.util.io.DisabledOutputStream;
|
import org.eclipse.jgit.util.io.DisabledOutputStream;
|
||||||
import org.kohsuke.args4j.Option;
|
import org.kohsuke.args4j.Option;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
@ -108,24 +103,24 @@ public class Reindex extends SiteProgram {
|
|||||||
|
|
||||||
private Injector dbInjector;
|
private Injector dbInjector;
|
||||||
private Injector sysInjector;
|
private Injector sysInjector;
|
||||||
private SitePaths sitePaths;
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int run() throws Exception {
|
public int run() throws Exception {
|
||||||
mustHaveValidSite();
|
mustHaveValidSite();
|
||||||
dbInjector = createDbInjector(MULTI_USER);
|
dbInjector = createDbInjector(MULTI_USER);
|
||||||
if (!IndexModule.isEnabled(dbInjector)) {
|
if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) {
|
||||||
throw die("Secondary index not enabled");
|
throw die("index.type must be configured (or not SQL)");
|
||||||
}
|
}
|
||||||
|
|
||||||
LifecycleManager dbManager = new LifecycleManager();
|
LifecycleManager dbManager = new LifecycleManager();
|
||||||
dbManager.add(dbInjector);
|
dbManager.add(dbInjector);
|
||||||
dbManager.start();
|
dbManager.start();
|
||||||
sitePaths = dbInjector.getInstance(SitePaths.class);
|
|
||||||
|
sysInjector = createSysInjector();
|
||||||
|
|
||||||
// Delete before any index may be created depending on this data.
|
// Delete before any index may be created depending on this data.
|
||||||
deleteAll();
|
deleteAll();
|
||||||
|
|
||||||
sysInjector = createSysInjector();
|
|
||||||
LifecycleManager sysManager = new LifecycleManager();
|
LifecycleManager sysManager = new LifecycleManager();
|
||||||
sysManager.add(sysInjector);
|
sysManager.add(sysInjector);
|
||||||
sysManager.start();
|
sysManager.start();
|
||||||
@ -141,7 +136,18 @@ public class Reindex extends SiteProgram {
|
|||||||
private Injector createSysInjector() {
|
private Injector createSysInjector() {
|
||||||
List<Module> modules = Lists.newArrayList();
|
List<Module> modules = Lists.newArrayList();
|
||||||
modules.add(PatchListCacheImpl.module());
|
modules.add(PatchListCacheImpl.module());
|
||||||
modules.add(new LuceneIndexModule(false, threads, dryRun));
|
AbstractModule changeIndexModule;
|
||||||
|
switch (IndexModule.getIndexType(dbInjector)) {
|
||||||
|
case LUCENE:
|
||||||
|
changeIndexModule = new LuceneIndexModule(false, threads, dryRun);
|
||||||
|
break;
|
||||||
|
case SOLR:
|
||||||
|
changeIndexModule = new SolrIndexModule(false, threads);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
changeIndexModule = new NoIndexModule();
|
||||||
|
}
|
||||||
|
modules.add(changeIndexModule);
|
||||||
modules.add(new ReviewDbModule());
|
modules.add(new ReviewDbModule());
|
||||||
modules.add(new AbstractModule() {
|
modules.add(new AbstractModule() {
|
||||||
@SuppressWarnings("rawtypes")
|
@SuppressWarnings("rawtypes")
|
||||||
@ -202,19 +208,8 @@ public class Reindex extends SiteProgram {
|
|||||||
if (dryRun) {
|
if (dryRun) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for (String index : SCHEMA_VERSIONS.keySet()) {
|
ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
|
||||||
File file = new File(sitePaths.index_dir, index);
|
index.deleteAll();
|
||||||
if (file.exists()) {
|
|
||||||
Directory dir = FSDirectory.open(file);
|
|
||||||
try {
|
|
||||||
for (String name : dir.listAll()) {
|
|
||||||
dir.deleteFile(name);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
dir.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private int indexAll() throws Exception {
|
private int indexAll() throws Exception {
|
||||||
@ -445,14 +440,7 @@ public class Reindex extends SiteProgram {
|
|||||||
if (dryRun) {
|
if (dryRun) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
FileBasedConfig cfg =
|
ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
|
||||||
new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
|
index.finishIndex();
|
||||||
cfg.load();
|
|
||||||
|
|
||||||
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
|
|
||||||
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
|
|
||||||
}
|
|
||||||
cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
|
|
||||||
cfg.save();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -21,6 +21,8 @@ import com.google.gerrit.server.query.QueryParseException;
|
|||||||
import com.google.gerrit.server.query.change.ChangeData;
|
import com.google.gerrit.server.query.change.ChangeData;
|
||||||
import com.google.gerrit.server.query.change.ChangeDataSource;
|
import com.google.gerrit.server.query.change.ChangeDataSource;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.ConfigInvalidException;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -51,11 +53,21 @@ public interface ChangeIndex {
|
|||||||
return Futures.immediateFuture(null);
|
return Futures.immediateFuture(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteAll() throws IOException {
|
||||||
|
// Do nothing.
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
||||||
throws QueryParseException {
|
throws QueryParseException {
|
||||||
throw new UnsupportedOperationException();
|
throw new UnsupportedOperationException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void finishIndex() {
|
||||||
|
// Do nothing.
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -94,6 +106,13 @@ public interface ChangeIndex {
|
|||||||
*/
|
*/
|
||||||
public ListenableFuture<Void> delete(ChangeData cd) throws IOException;
|
public ListenableFuture<Void> delete(ChangeData cd) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete all change documents from the index.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public void deleteAll() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Convert the given operator predicate into a source searching the index and
|
* Convert the given operator predicate into a source searching the index and
|
||||||
* returning only the documents matching that predicate.
|
* returning only the documents matching that predicate.
|
||||||
@ -108,4 +127,13 @@ public interface ChangeIndex {
|
|||||||
*/
|
*/
|
||||||
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
||||||
throws QueryParseException;
|
throws QueryParseException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark completion of indexing.
|
||||||
|
*
|
||||||
|
* @throws ConfigInvalidException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public void finishIndex() throws IOException,
|
||||||
|
ConfigInvalidException;
|
||||||
}
|
}
|
||||||
|
@ -36,9 +36,15 @@ import org.eclipse.jgit.lib.Config;
|
|||||||
* implementations (e.g. Lucene).
|
* implementations (e.g. Lucene).
|
||||||
*/
|
*/
|
||||||
public class IndexModule extends AbstractModule {
|
public class IndexModule extends AbstractModule {
|
||||||
public static boolean isEnabled(Injector injector) {
|
public enum IndexType {
|
||||||
return injector.getInstance(Key.get(Config.class, GerritServerConfig.class))
|
SQL, LUCENE, SOLR;
|
||||||
.getBoolean("index", null, "enabled", false);
|
}
|
||||||
|
|
||||||
|
/** Type of secondary index. */
|
||||||
|
public static IndexType getIndexType(Injector injector) {
|
||||||
|
Config cfg = injector.getInstance(
|
||||||
|
Key.get(Config.class, GerritServerConfig.class));
|
||||||
|
return cfg.getEnum("index", null, "type", IndexType.SQL);
|
||||||
}
|
}
|
||||||
|
|
||||||
private final int threads;
|
private final int threads;
|
||||||
|
@ -56,11 +56,21 @@ public class IndexRewriteTest extends TestCase {
|
|||||||
throw new UnsupportedOperationException();
|
throw new UnsupportedOperationException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteAll() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
||||||
throws QueryParseException {
|
throws QueryParseException {
|
||||||
return new Source();
|
return new Source();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void finishIndex() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class Source implements ChangeDataSource {
|
private static class Source implements ChangeDataSource {
|
||||||
|
19
gerrit-solr/BUCK
Normal file
19
gerrit-solr/BUCK
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
java_library(
|
||||||
|
name = 'solr',
|
||||||
|
srcs = glob(['src/main/java/**/*.java']),
|
||||||
|
deps = [
|
||||||
|
'//gerrit-antlr:query_exception',
|
||||||
|
'//gerrit-extension-api:api',
|
||||||
|
'//gerrit-lucene:query_builder',
|
||||||
|
'//gerrit-reviewdb:client',
|
||||||
|
'//gerrit-server:server',
|
||||||
|
'//lib:guava',
|
||||||
|
'//lib:gwtorm',
|
||||||
|
'//lib/guice:guice',
|
||||||
|
'//lib/jgit:jgit',
|
||||||
|
'//lib/log:api',
|
||||||
|
'//lib/lucene:core',
|
||||||
|
'//lib/solr:solrj',
|
||||||
|
],
|
||||||
|
visibility = ['PUBLIC'],
|
||||||
|
)
|
@ -0,0 +1,80 @@
|
|||||||
|
// Copyright (C) 2013 The Android Open Source Project
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.package com.google.gerrit.server.git;
|
||||||
|
|
||||||
|
package com.google.gerrit.solr;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
import com.google.gerrit.extensions.events.LifecycleListener;
|
||||||
|
import com.google.gerrit.server.config.SitePaths;
|
||||||
|
import com.google.gerrit.server.index.ChangeField;
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import com.google.inject.ProvisionException;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.ConfigInvalidException;
|
||||||
|
import org.eclipse.jgit.storage.file.FileBasedConfig;
|
||||||
|
import org.eclipse.jgit.util.FS;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
class IndexVersionCheck implements LifecycleListener {
|
||||||
|
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
|
||||||
|
SolrChangeIndex.CHANGES_OPEN, ChangeField.SCHEMA_VERSION,
|
||||||
|
SolrChangeIndex.CHANGES_CLOSED, ChangeField.SCHEMA_VERSION);
|
||||||
|
|
||||||
|
public static File solrIndexConfig(SitePaths sitePaths) {
|
||||||
|
return new File(sitePaths.index_dir, "gerrit_index.config");
|
||||||
|
}
|
||||||
|
|
||||||
|
private final SitePaths sitePaths;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
IndexVersionCheck(SitePaths sitePaths) {
|
||||||
|
this.sitePaths = sitePaths;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void start() {
|
||||||
|
// TODO Query schema version from a special meta-document
|
||||||
|
File file = solrIndexConfig(sitePaths);
|
||||||
|
try {
|
||||||
|
FileBasedConfig cfg = new FileBasedConfig(file, FS.detect());
|
||||||
|
cfg.load();
|
||||||
|
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
|
||||||
|
int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
|
||||||
|
if (schemaVersion != e.getValue()) {
|
||||||
|
throw new ProvisionException(String.format(
|
||||||
|
"wrong index schema version for \"%s\": expected %d, found %d%s",
|
||||||
|
e.getKey(), e.getValue(), schemaVersion, upgrade()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ProvisionException("unable to read " + file);
|
||||||
|
} catch (ConfigInvalidException e) {
|
||||||
|
throw new ProvisionException("invalid config file " + file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stop() {
|
||||||
|
// Do nothing.
|
||||||
|
}
|
||||||
|
|
||||||
|
private final String upgrade() {
|
||||||
|
return "\nRun reindex to rebuild the index:\n"
|
||||||
|
+ "$ java -jar gerrit.war reindex -d "
|
||||||
|
+ sitePaths.site_path.getAbsolutePath();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,318 @@
|
|||||||
|
// Copyright (C) 2013 The Android Open Source Project
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.package com.google.gerrit.server.git;
|
||||||
|
|
||||||
|
package com.google.gerrit.solr;
|
||||||
|
|
||||||
|
import static com.google.gerrit.server.query.change.IndexRewriteImpl.CLOSED_STATUSES;
|
||||||
|
import static com.google.gerrit.server.query.change.IndexRewriteImpl.OPEN_STATUSES;
|
||||||
|
import static com.google.gerrit.solr.IndexVersionCheck.SCHEMA_VERSIONS;
|
||||||
|
import static com.google.gerrit.solr.IndexVersionCheck.solrIndexConfig;
|
||||||
|
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import com.google.common.collect.Sets;
|
||||||
|
import com.google.common.util.concurrent.Futures;
|
||||||
|
import com.google.common.util.concurrent.ListenableFuture;
|
||||||
|
import com.google.gerrit.extensions.events.LifecycleListener;
|
||||||
|
import com.google.gerrit.lucene.QueryBuilder;
|
||||||
|
import com.google.gerrit.reviewdb.client.Change;
|
||||||
|
import com.google.gerrit.server.config.GerritServerConfig;
|
||||||
|
import com.google.gerrit.server.config.SitePaths;
|
||||||
|
import com.google.gerrit.server.index.ChangeField;
|
||||||
|
import com.google.gerrit.server.index.ChangeIndex;
|
||||||
|
import com.google.gerrit.server.index.FieldDef;
|
||||||
|
import com.google.gerrit.server.index.FieldDef.FillArgs;
|
||||||
|
import com.google.gerrit.server.index.FieldType;
|
||||||
|
import com.google.gerrit.server.query.Predicate;
|
||||||
|
import com.google.gerrit.server.query.QueryParseException;
|
||||||
|
import com.google.gerrit.server.query.change.ChangeData;
|
||||||
|
import com.google.gerrit.server.query.change.ChangeDataSource;
|
||||||
|
import com.google.gerrit.server.query.change.IndexRewriteImpl;
|
||||||
|
import com.google.gwtorm.server.OrmException;
|
||||||
|
import com.google.gwtorm.server.ResultSet;
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import com.google.inject.Singleton;
|
||||||
|
|
||||||
|
import org.apache.lucene.search.Query;
|
||||||
|
import org.apache.solr.client.solrj.SolrQuery;
|
||||||
|
import org.apache.solr.client.solrj.SolrServer;
|
||||||
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
||||||
|
import org.apache.solr.common.SolrDocument;
|
||||||
|
import org.apache.solr.common.SolrDocumentList;
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
import org.eclipse.jgit.errors.ConfigInvalidException;
|
||||||
|
import org.eclipse.jgit.lib.Config;
|
||||||
|
import org.eclipse.jgit.storage.file.FileBasedConfig;
|
||||||
|
import org.eclipse.jgit.util.FS;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/** Secondary index implementation using a remote Solr instance. */
|
||||||
|
@Singleton
|
||||||
|
class SolrChangeIndex implements ChangeIndex, LifecycleListener {
|
||||||
|
public static final String CHANGES_OPEN = "changes_open";
|
||||||
|
public static final String CHANGES_CLOSED = "changes_closed";
|
||||||
|
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
|
||||||
|
|
||||||
|
private final FillArgs fillArgs;
|
||||||
|
private final SitePaths sitePaths;
|
||||||
|
private final CloudSolrServer openIndex;
|
||||||
|
private final CloudSolrServer closedIndex;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
SolrChangeIndex(
|
||||||
|
@GerritServerConfig Config cfg,
|
||||||
|
FillArgs fillArgs,
|
||||||
|
SitePaths sitePaths) throws IOException {
|
||||||
|
this.fillArgs = fillArgs;
|
||||||
|
this.sitePaths = sitePaths;
|
||||||
|
|
||||||
|
String url = cfg.getString("index", "solr", "url");
|
||||||
|
if (Strings.isNullOrEmpty(url)) {
|
||||||
|
throw new IllegalStateException("index.solr.url must be supplied");
|
||||||
|
}
|
||||||
|
|
||||||
|
openIndex = new CloudSolrServer(url);
|
||||||
|
openIndex.setDefaultCollection(CHANGES_OPEN);
|
||||||
|
|
||||||
|
closedIndex = new CloudSolrServer(url);
|
||||||
|
closedIndex.setDefaultCollection(CHANGES_CLOSED);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void start() {
|
||||||
|
// Do nothing.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stop() {
|
||||||
|
openIndex.shutdown();
|
||||||
|
closedIndex.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
|
||||||
|
String id = cd.getId().toString();
|
||||||
|
SolrInputDocument doc = toDocument(cd);
|
||||||
|
try {
|
||||||
|
if (cd.getChange().getStatus().isOpen()) {
|
||||||
|
closedIndex.deleteById(id);
|
||||||
|
openIndex.add(doc);
|
||||||
|
} else {
|
||||||
|
openIndex.deleteById(id);
|
||||||
|
closedIndex.add(doc);
|
||||||
|
}
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
commit(openIndex);
|
||||||
|
commit(closedIndex);
|
||||||
|
return Futures.immediateFuture(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ListenableFuture<Void> replace(ChangeData cd) throws IOException {
|
||||||
|
String id = cd.getId().toString();
|
||||||
|
SolrInputDocument doc = toDocument(cd);
|
||||||
|
try {
|
||||||
|
if (cd.getChange().getStatus().isOpen()) {
|
||||||
|
closedIndex.deleteById(id);
|
||||||
|
openIndex.add(doc);
|
||||||
|
} else {
|
||||||
|
openIndex.deleteById(id);
|
||||||
|
closedIndex.add(doc);
|
||||||
|
}
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
commit(openIndex);
|
||||||
|
commit(closedIndex);
|
||||||
|
return Futures.immediateFuture(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ListenableFuture<Void> delete(ChangeData cd) throws IOException {
|
||||||
|
String id = cd.getId().toString();
|
||||||
|
try {
|
||||||
|
if (cd.getChange().getStatus().isOpen()) {
|
||||||
|
openIndex.deleteById(id);
|
||||||
|
commit(openIndex);
|
||||||
|
} else {
|
||||||
|
closedIndex.deleteById(id);
|
||||||
|
commit(closedIndex);
|
||||||
|
}
|
||||||
|
return Futures.immediateFuture(null);
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteAll() throws IOException {
|
||||||
|
try {
|
||||||
|
openIndex.deleteByQuery("*:*");
|
||||||
|
closedIndex.deleteByQuery("*:*");
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
commit(openIndex);
|
||||||
|
commit(closedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChangeDataSource getSource(Predicate<ChangeData> p)
|
||||||
|
throws QueryParseException {
|
||||||
|
Set<Change.Status> statuses = IndexRewriteImpl.getPossibleStatus(p);
|
||||||
|
List<SolrServer> indexes = Lists.newArrayListWithCapacity(2);
|
||||||
|
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
|
||||||
|
indexes.add(openIndex);
|
||||||
|
}
|
||||||
|
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
|
||||||
|
indexes.add(closedIndex);
|
||||||
|
}
|
||||||
|
return new QuerySource(indexes, QueryBuilder.toQuery(p));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void commit(SolrServer server) throws IOException {
|
||||||
|
try {
|
||||||
|
server.commit();
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class QuerySource implements ChangeDataSource {
|
||||||
|
private final List<SolrServer> indexes;
|
||||||
|
private final SolrQuery query;
|
||||||
|
|
||||||
|
public QuerySource(List<SolrServer> indexes, Query q) {
|
||||||
|
this.indexes = indexes;
|
||||||
|
|
||||||
|
query = new SolrQuery(q.toString());
|
||||||
|
query.setParam("shards.tolerant", true);
|
||||||
|
query.setFields(ID_FIELD);
|
||||||
|
query.setSort(
|
||||||
|
ChangeField.UPDATED.getName(),
|
||||||
|
SolrQuery.ORDER.desc);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getCardinality() {
|
||||||
|
return 10; // TODO: estimate from solr?
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasChange() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ResultSet<ChangeData> read() throws OrmException {
|
||||||
|
try {
|
||||||
|
// TODO Sort documents during merge to select only top N.
|
||||||
|
SolrDocumentList docs = new SolrDocumentList();
|
||||||
|
for (SolrServer index : indexes) {
|
||||||
|
docs.addAll(index.query(query).getResults());
|
||||||
|
}
|
||||||
|
|
||||||
|
List<ChangeData> result = Lists.newArrayListWithCapacity(docs.size());
|
||||||
|
for (SolrDocument doc : docs) {
|
||||||
|
Integer v = (Integer) doc.getFieldValue(ID_FIELD);
|
||||||
|
result.add(new ChangeData(new Change.Id(v.intValue())));
|
||||||
|
}
|
||||||
|
|
||||||
|
final List<ChangeData> r = Collections.unmodifiableList(result);
|
||||||
|
return new ResultSet<ChangeData>() {
|
||||||
|
@Override
|
||||||
|
public Iterator<ChangeData> iterator() {
|
||||||
|
return r.iterator();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ChangeData> toList() {
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// Do nothing.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
throw new OrmException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private SolrInputDocument toDocument(ChangeData cd) throws IOException {
|
||||||
|
try {
|
||||||
|
SolrInputDocument result = new SolrInputDocument();
|
||||||
|
for (FieldDef<ChangeData, ?> f : ChangeField.ALL.values()) {
|
||||||
|
if (f.isRepeatable()) {
|
||||||
|
add(result, f, (Iterable<?>) f.get(cd, fillArgs));
|
||||||
|
} else {
|
||||||
|
add(result, f, Collections.singleton(f.get(cd, fillArgs)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} catch (OrmException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void add(SolrInputDocument doc, FieldDef<ChangeData, ?> f,
|
||||||
|
Iterable<?> values) throws OrmException {
|
||||||
|
if (f.getType() == FieldType.INTEGER) {
|
||||||
|
for (Object value : values) {
|
||||||
|
doc.addField(f.getName(), (Integer) value);
|
||||||
|
}
|
||||||
|
} else if (f.getType() == FieldType.LONG) {
|
||||||
|
for (Object value : values) {
|
||||||
|
doc.addField(f.getName(), (Long) value);
|
||||||
|
}
|
||||||
|
} else if (f.getType() == FieldType.TIMESTAMP) {
|
||||||
|
for (Object v : values) {
|
||||||
|
doc.addField(f.getName(), QueryBuilder.toIndexTime((Timestamp) v));
|
||||||
|
}
|
||||||
|
} else if (f.getType() == FieldType.EXACT
|
||||||
|
|| f.getType() == FieldType.PREFIX
|
||||||
|
|| f.getType() == FieldType.FULL_TEXT) {
|
||||||
|
for (Object value : values) {
|
||||||
|
doc.addField(f.getName(), (String) value);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw QueryBuilder.badFieldType(f.getType());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void finishIndex() throws IOException,
|
||||||
|
ConfigInvalidException {
|
||||||
|
// TODO Move the schema version information to a special meta-document
|
||||||
|
FileBasedConfig cfg = new FileBasedConfig(
|
||||||
|
solrIndexConfig(sitePaths),
|
||||||
|
FS.detect());
|
||||||
|
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
|
||||||
|
cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
|
||||||
|
}
|
||||||
|
cfg.save();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,43 @@
|
|||||||
|
// Copyright (C) 2013 The Android Open Source Project
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.package com.google.gerrit.server.git;
|
||||||
|
|
||||||
|
package com.google.gerrit.solr;
|
||||||
|
|
||||||
|
import com.google.gerrit.lifecycle.LifecycleModule;
|
||||||
|
import com.google.gerrit.server.index.ChangeIndex;
|
||||||
|
import com.google.gerrit.server.index.IndexModule;
|
||||||
|
|
||||||
|
public class SolrIndexModule extends LifecycleModule {
|
||||||
|
private final boolean checkVersion;
|
||||||
|
private final int threads;
|
||||||
|
|
||||||
|
public SolrIndexModule() {
|
||||||
|
this(true, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public SolrIndexModule(boolean checkVersion, int threads) {
|
||||||
|
this.checkVersion = checkVersion;
|
||||||
|
this.threads = threads;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void configure() {
|
||||||
|
install(new IndexModule(threads));
|
||||||
|
bind(ChangeIndex.class).to(SolrChangeIndex.class);
|
||||||
|
listener().to(SolrChangeIndex.class);
|
||||||
|
if (checkVersion) {
|
||||||
|
listener().to(IndexVersionCheck.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -10,6 +10,7 @@ java_library2(
|
|||||||
'//gerrit-reviewdb:server',
|
'//gerrit-reviewdb:server',
|
||||||
'//gerrit-server:common_rules',
|
'//gerrit-server:common_rules',
|
||||||
'//gerrit-server:server',
|
'//gerrit-server:server',
|
||||||
|
'//gerrit-solr:solr',
|
||||||
'//gerrit-sshd:sshd',
|
'//gerrit-sshd:sshd',
|
||||||
'//lib:gwtorm',
|
'//lib:gwtorm',
|
||||||
'//lib/guice:guice',
|
'//lib/guice:guice',
|
||||||
|
@ -50,6 +50,7 @@ import com.google.gerrit.server.schema.DataSourceType;
|
|||||||
import com.google.gerrit.server.schema.DatabaseModule;
|
import com.google.gerrit.server.schema.DatabaseModule;
|
||||||
import com.google.gerrit.server.schema.SchemaModule;
|
import com.google.gerrit.server.schema.SchemaModule;
|
||||||
import com.google.gerrit.server.schema.SchemaVersionCheck;
|
import com.google.gerrit.server.schema.SchemaVersionCheck;
|
||||||
|
import com.google.gerrit.solr.SolrIndexModule;
|
||||||
import com.google.gerrit.sshd.SshKeyCacheImpl;
|
import com.google.gerrit.sshd.SshKeyCacheImpl;
|
||||||
import com.google.gerrit.sshd.SshModule;
|
import com.google.gerrit.sshd.SshModule;
|
||||||
import com.google.gerrit.sshd.commands.MasterCommandModule;
|
import com.google.gerrit.sshd.commands.MasterCommandModule;
|
||||||
@ -238,11 +239,18 @@ public class WebAppInitializer extends GuiceServletContextListener {
|
|||||||
modules.add(new SmtpEmailSender.Module());
|
modules.add(new SmtpEmailSender.Module());
|
||||||
modules.add(new SignedTokenEmailTokenVerifier.Module());
|
modules.add(new SignedTokenEmailTokenVerifier.Module());
|
||||||
modules.add(new PluginModule());
|
modules.add(new PluginModule());
|
||||||
if (IndexModule.isEnabled(cfgInjector)) {
|
AbstractModule changeIndexModule;
|
||||||
modules.add(new LuceneIndexModule());
|
switch (IndexModule.getIndexType(cfgInjector)) {
|
||||||
} else {
|
case LUCENE:
|
||||||
modules.add(new NoIndexModule());
|
changeIndexModule = new LuceneIndexModule();
|
||||||
|
break;
|
||||||
|
case SOLR:
|
||||||
|
changeIndexModule = new SolrIndexModule();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
changeIndexModule = new NoIndexModule();
|
||||||
}
|
}
|
||||||
|
modules.add(changeIndexModule);
|
||||||
modules.add(new CanonicalWebUrlModule() {
|
modules.add(new CanonicalWebUrlModule() {
|
||||||
@Override
|
@Override
|
||||||
protected Class<? extends Provider<String>> provider() {
|
protected Class<? extends Provider<String>> provider() {
|
||||||
|
24
lib/BUCK
24
lib/BUCK
@ -246,27 +246,3 @@ maven_jar(
|
|||||||
visibility = ['//lib:easymock'],
|
visibility = ['//lib:easymock'],
|
||||||
attach_source = False,
|
attach_source = False,
|
||||||
)
|
)
|
||||||
|
|
||||||
maven_jar(
|
|
||||||
name = 'lucene-core',
|
|
||||||
id = 'org.apache.lucene:lucene-core:4.3.0',
|
|
||||||
bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3',
|
|
||||||
src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a',
|
|
||||||
license = 'Apache2.0',
|
|
||||||
exclude = [
|
|
||||||
'META-INF/LICENSE.txt',
|
|
||||||
'META-INF/NOTICE.txt',
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
maven_jar(
|
|
||||||
name = 'lucene-analyzers-common',
|
|
||||||
id = 'org.apache.lucene:lucene-analyzers-common:4.3.0',
|
|
||||||
bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56',
|
|
||||||
src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b',
|
|
||||||
license = 'Apache2.0',
|
|
||||||
exclude = [
|
|
||||||
'META-INF/LICENSE.txt',
|
|
||||||
'META-INF/NOTICE.txt',
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
@ -15,7 +15,10 @@ maven_jar(
|
|||||||
license = 'Apache2.0',
|
license = 'Apache2.0',
|
||||||
exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
|
exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
|
||||||
attach_source = False,
|
attach_source = False,
|
||||||
visibility = ['//lib:velocity'],
|
visibility = [
|
||||||
|
'//lib:velocity',
|
||||||
|
'//lib/solr:zookeeper',
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
maven_jar(
|
maven_jar(
|
||||||
@ -39,19 +42,6 @@ maven_jar(
|
|||||||
exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
|
exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
|
||||||
)
|
)
|
||||||
|
|
||||||
maven_jar(
|
|
||||||
name = 'logging',
|
|
||||||
id = 'commons-logging:commons-logging:1.1.1',
|
|
||||||
sha1 = '5043bfebc3db072ed80fbd362e7caf00e885d8ae',
|
|
||||||
license = 'Apache2.0',
|
|
||||||
exclude = [
|
|
||||||
'META-INF/LICENSE',
|
|
||||||
'META-INF/NOTICE',
|
|
||||||
],
|
|
||||||
attach_source = False,
|
|
||||||
visibility = ['//lib/openid:'],
|
|
||||||
)
|
|
||||||
|
|
||||||
maven_jar(
|
maven_jar(
|
||||||
name = 'net',
|
name = 'net',
|
||||||
id = 'commons-net:commons-net:2.2',
|
id = 'commons-net:commons-net:2.2',
|
||||||
@ -77,3 +67,39 @@ maven_jar(
|
|||||||
attach_source = False,
|
attach_source = False,
|
||||||
exclude = ['META-INF/LICENSE'],
|
exclude = ['META-INF/LICENSE'],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'io',
|
||||||
|
id = 'commons-io:commons-io:1.4',
|
||||||
|
sha1 = 'a8762d07e76cfde2395257a5da47ba7c1dbd3dce',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'httpclient',
|
||||||
|
id = 'org.apache.httpcomponents:httpclient:4.2.5',
|
||||||
|
bin_sha1 = '666e26e76f2e87d84e4f16acb546481ae1b8e9a6',
|
||||||
|
src_sha1 = '55d345272944d7e8dace47925336a3764ee0e24b',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
deps = [
|
||||||
|
':codec',
|
||||||
|
':httpcore',
|
||||||
|
'//lib/log:jcl-over-slf4j',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'httpcore',
|
||||||
|
id = 'org.apache.httpcomponents:httpcore:4.2.4',
|
||||||
|
bin_sha1 = '3b7f38df6de5dd8b500e602ae8c2dd5ee446f883',
|
||||||
|
src_sha1 = 'c3ffe3a73348645042fb0b06303b6a3de194494e',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'httpmime',
|
||||||
|
id = 'org.apache.httpcomponents:httpmime:4.2.5',
|
||||||
|
bin_sha1 = '412b9914d0adec6d5716df1ada8acbc4f6f2dd37',
|
||||||
|
src_sha1 = 'c07ce7f6b153284a9ebaf58532c2442200cf3aa2',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
)
|
||||||
|
@ -22,3 +22,10 @@ maven_jar(
|
|||||||
license = 'Apache2.0',
|
license = 'Apache2.0',
|
||||||
exclude = ['META-INF/LICENSE', 'META-INF/NOTICE'],
|
exclude = ['META-INF/LICENSE', 'META-INF/NOTICE'],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'jcl-over-slf4j',
|
||||||
|
id = 'org.slf4j:jcl-over-slf4j:1.6.1',
|
||||||
|
sha1 = '99c61095a14dfc9e47a086068033c286bf236475',
|
||||||
|
license = 'slf4j',
|
||||||
|
)
|
||||||
|
49
lib/lucene/BUCK
Normal file
49
lib/lucene/BUCK
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
include_defs('//lib/maven.defs')
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'core',
|
||||||
|
id = 'org.apache.lucene:lucene-core:4.3.0',
|
||||||
|
bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3',
|
||||||
|
src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
exclude = [
|
||||||
|
'META-INF/LICENSE.txt',
|
||||||
|
'META-INF/NOTICE.txt',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'analyzers-common',
|
||||||
|
id = 'org.apache.lucene:lucene-analyzers-common:4.3.0',
|
||||||
|
bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56',
|
||||||
|
src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
exclude = [
|
||||||
|
'META-INF/LICENSE.txt',
|
||||||
|
'META-INF/NOTICE.txt',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'highlighter',
|
||||||
|
id = 'org.apache.lucene:lucene-highlighter:4.3.0',
|
||||||
|
bin_sha1 = '9e6d60921e16a0d6b2e609c6a02a8b08cd7f643c',
|
||||||
|
src_sha1 = '0ff70cae1a8fb7af29bf254d90e9885961deed5e',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'queries',
|
||||||
|
id = 'org.apache.lucene:lucene-queries:4.3.0',
|
||||||
|
bin_sha1 = '68e01022bdf4f869b95362c9af964846e5d3cf2d',
|
||||||
|
src_sha1 = '3e3541c1b9f44c532ce88ab6a12216566c3399df',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'spellchecker',
|
||||||
|
id = 'org.apache.lucene:lucene-spellchecker:3.6.2',
|
||||||
|
bin_sha1 = '15db0c0cfee44e275f15ad046e46b9a05910ad24',
|
||||||
|
src_sha1 = 'bbecb3fb725ae594101c165a72c102296007c203',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
)
|
@ -6,10 +6,10 @@ maven_jar(
|
|||||||
sha1 = 'de4f1b33d3b0f0b2ab1d32834ec1190b39db4160',
|
sha1 = 'de4f1b33d3b0f0b2ab1d32834ec1190b39db4160',
|
||||||
license = 'Apache2.0',
|
license = 'Apache2.0',
|
||||||
deps = [
|
deps = [
|
||||||
':httpclient',
|
|
||||||
':nekohtml',
|
':nekohtml',
|
||||||
':xerces',
|
':xerces',
|
||||||
'//lib/commons:logging',
|
'//lib/commons:httpclient',
|
||||||
|
'//lib/log:jcl-over-slf4j',
|
||||||
'//lib/guice:guice',
|
'//lib/guice:guice',
|
||||||
],
|
],
|
||||||
visibility = ['PUBLIC'],
|
visibility = ['PUBLIC'],
|
||||||
@ -33,31 +33,3 @@ maven_jar(
|
|||||||
attach_source = False,
|
attach_source = False,
|
||||||
visibility = [],
|
visibility = [],
|
||||||
)
|
)
|
||||||
|
|
||||||
maven_jar(
|
|
||||||
name = 'httpclient',
|
|
||||||
id = 'org.apache.httpcomponents:httpclient:4.1',
|
|
||||||
sha1 = '93cd011acb220de08b57d96106e5800d7097742b',
|
|
||||||
license = 'Apache2.0',
|
|
||||||
deps = [
|
|
||||||
':httpcore',
|
|
||||||
'//lib/commons:codec',
|
|
||||||
'//lib/commons:logging',
|
|
||||||
],
|
|
||||||
exclude = [
|
|
||||||
'META-INF/LICENSE.txt',
|
|
||||||
'META-INF/NOTICE.txt',
|
|
||||||
],
|
|
||||||
visibility = ['//gerrit-acceptance-tests:'],
|
|
||||||
)
|
|
||||||
|
|
||||||
maven_jar(
|
|
||||||
name = 'httpcore',
|
|
||||||
id = 'org.apache.httpcomponents:httpcore:4.1',
|
|
||||||
sha1 = '33fc26c02f8043ab0ede19eadc8c9885386b255c',
|
|
||||||
license = 'Apache2.0',
|
|
||||||
exclude = [
|
|
||||||
'META-INF/LICENSE.txt',
|
|
||||||
'META-INF/NOTICE.txt',
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
33
lib/solr/BUCK
Normal file
33
lib/solr/BUCK
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
include_defs('//lib/maven.defs')
|
||||||
|
|
||||||
|
# Java client library to use Solr over the network.
|
||||||
|
maven_jar(
|
||||||
|
name = 'solrj',
|
||||||
|
id = 'org.apache.solr:solr-solrj:4.3.1',
|
||||||
|
sha1 = '433fe37796e67eaeb4452f69eb1fae2de27cb7a8',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
deps = [
|
||||||
|
':noggit',
|
||||||
|
':zookeeper',
|
||||||
|
'//lib/commons:httpclient',
|
||||||
|
'//lib/commons:httpmime',
|
||||||
|
'//lib/commons:io',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'noggit',
|
||||||
|
id = 'org.noggit:noggit:0.5',
|
||||||
|
sha1 = '8e6e65624d2e09a30190c6434abe23b7d4e5413c',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
visibility = [],
|
||||||
|
)
|
||||||
|
|
||||||
|
maven_jar(
|
||||||
|
name = 'zookeeper',
|
||||||
|
id = 'org.apache.zookeeper:zookeeper:3.4.5',
|
||||||
|
sha1 = 'c0f69fb36526552a8f0bc548a6c33c49cf08e562',
|
||||||
|
license = 'Apache2.0',
|
||||||
|
deps = ['//lib/log:api'],
|
||||||
|
visibility = [],
|
||||||
|
)
|
Loading…
Reference in New Issue
Block a user