opendev/gerrit
Code Issues Proposed changes

Remove Solr

Browse Source 
It isn't used in the wild and is going to be replaced with
Elastic Search.

Change-Id: Ic62ae3103fca2214904dbf2faf4c861b5f0ae9b5
This commit is contained in:
David Ostrovsky 2014-07-12 17:04:24 +02:00 committed by David Pursehouse
parent b50a7edd46
commit 7fe1752bfd
13 changed files with 1 additions and 581 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Documentation/config-gerrit.txt
View File

@ -2289,10 +2289,6 @@ values are:
+
A link:http://lucene.apache.org/[Lucene] index is used.
+
* `SOLR`
+
A link:https://cwiki.apache.org/confluence/display/solr/SolrCloud[
SolrCloud] index is used.
+
By default, `LUCENE`.
@ -2413,17 +2409,6 @@ Sample Lucene index configuration:
maxBufferedDocs = 500
----
==== Solr configuration
Open and closed changes are indexed in separate indexes named
'changes_open' and 'changes_closed' respectively.
The following settings are only used when the index type is `SOLR`.
[[index.url]]index.url::
+
URL of the index server.
[[ldap]]
=== Section ldap

1
gerrit-pgm/BUCK
View File

@ -103,7 +103,6 @@ java_library(
'//gerrit-lucene:lucene',
'//gerrit-oauth:oauth',
'//gerrit-openid:openid',
'//gerrit-solr:solr',
'//lib:args4j',
'//lib:gwtorm',
'//lib:protobuf',

3
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java
View File

@ -77,7 +77,6 @@ import com.google.gerrit.server.securestore.SecureStoreProvider;
import com.google.gerrit.server.ssh.NoSshKeyCache;
import com.google.gerrit.server.ssh.NoSshModule;
import com.google.gerrit.server.ssh.SshAddressesModule;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.gerrit.sshd.SshHostKeyModule;
import com.google.gerrit.sshd.SshKeyCacheImpl;
import com.google.gerrit.sshd.SshModule;
@ -383,8 +382,6 @@ public class Daemon extends SiteProgram {
switch (indexType) {
case LUCENE:
return luceneModule != null ? luceneModule : new LuceneIndexModule();
case SOLR:
return new SolrIndexModule();
default:
throw new IllegalStateException("unsupported index.type = " + indexType);
}

4
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
View File

@ -35,7 +35,6 @@ import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.SiteIndexer;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
@ -122,9 +121,6 @@ public class Reindex extends SiteProgram {
case LUCENE:
changeIndexModule = new LuceneIndexModule(version, threads, outputBase);
break;
case SOLR:
changeIndexModule = new SolrIndexModule(false, threads, outputBase);
break;
default:
throw new IllegalStateException("unsupported index.type");
}

3
gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java
View File

@ -51,9 +51,6 @@ class InitIndex implements InitStep {
ui.header("Index");
IndexType type = index.select("Type", "type", IndexType.LUCENE);
if (type == IndexType.SOLR) {
index.string("Solr Index URL", "url", "localhost:9983");
}
if (site.isNew && type == IndexType.LUCENE) {
LuceneChangeIndex.setReady(
site, ChangeSchemas.getLatest().getVersion(), true);

2
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
View File

@ -38,7 +38,7 @@ import org.eclipse.jgit.lib.Config;
*/
public class IndexModule extends LifecycleModule {
public enum IndexType {
LUCENE, SOLR
LUCENE
}
/** Type of secondary index. */

20
gerrit-solr/BUCK
View File

@ -1,20 +0,0 @@
java_library(
name = 'solr',
srcs = glob(['src/main/java/**/*.java']),
deps = [
'//gerrit-antlr:query_exception',
'//gerrit-extension-api:api',
'//gerrit-lucene:query_builder',
'//gerrit-reviewdb:server',
'//gerrit-server:server',
'//lib:guava',
'//lib:gwtorm',
'//lib/guice:guice',
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:analyzers-common',
'//lib/lucene:core',
'//lib/solr:solrj',
],
visibility = ['PUBLIC'],
)

80
gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java
View File

@ -1,80 +0,0 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.solr;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatest().getVersion(),
SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatest().getVersion());
public static Path solrIndexConfig(SitePaths sitePaths) {
return sitePaths.index_dir.resolve("gerrit_index.config");
}
private final SitePaths sitePaths;
@Inject
IndexVersionCheck(SitePaths sitePaths) {
this.sitePaths = sitePaths;
}
@Override
public void start() {
// TODO Query schema version from a special meta-document
Path path = solrIndexConfig(sitePaths);
try {
FileBasedConfig cfg = new FileBasedConfig(path.toFile(), FS.detect());
cfg.load();
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
if (schemaVersion != e.getValue()) {
throw new ProvisionException(String.format(
"wrong index schema version for \"%s\": expected %d, found %d%s",
e.getKey(), e.getValue(), schemaVersion, upgrade()));
}
}
} catch (IOException e) {
throw new ProvisionException("unable to read " + path);
} catch (ConfigInvalidException e) {
throw new ProvisionException("invalid config file " + path);
}
}
@Override
public void stop() {
// Do nothing.
}
private final String upgrade() {
return "\nRun reindex to rebuild the index:\n"
+ "$ java -jar gerrit.war reindex -d "
+ sitePaths.site_path.toAbsolutePath();
}
}

338
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
View File

@ -1,338 +0,0 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.solr;
import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
import static com.google.gerrit.solr.IndexVersionCheck.SCHEMA_VERSIONS;
import static com.google.gerrit.solr.IndexVersionCheck.solrIndexConfig;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lucene.QueryBuilder;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexRewriteImpl;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Provider;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.search.Query;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.SortClause;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Secondary index implementation using a remote Solr instance. */
class SolrChangeIndex implements ChangeIndex, LifecycleListener {
public static final String CHANGES_OPEN = "changes_open";
public static final String CHANGES_CLOSED = "changes_closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
private final Provider<ReviewDb> db;
private final ChangeData.Factory changeDataFactory;
private final FillArgs fillArgs;
private final SitePaths sitePaths;
private final IndexCollection indexes;
private final CloudSolrServer openIndex;
private final CloudSolrServer closedIndex;
private final Schema<ChangeData> schema;
private final QueryBuilder queryBuilder;
SolrChangeIndex(
@GerritServerConfig Config cfg,
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
FillArgs fillArgs,
SitePaths sitePaths,
IndexCollection indexes,
Schema<ChangeData> schema,
String base) throws IOException {
this.db = db;
this.changeDataFactory = changeDataFactory;
this.fillArgs = fillArgs;
this.sitePaths = sitePaths;
this.indexes = indexes;
this.schema = schema;
String url = cfg.getString("index", null, "url");
if (Strings.isNullOrEmpty(url)) {
throw new IllegalStateException("index.url must be supplied");
}
queryBuilder = new QueryBuilder(
new StandardAnalyzer(CharArraySet.EMPTY_SET));
base = Strings.nullToEmpty(base);
openIndex = new CloudSolrServer(url);
openIndex.setDefaultCollection(base + CHANGES_OPEN);
closedIndex = new CloudSolrServer(url);
closedIndex.setDefaultCollection(base + CHANGES_CLOSED);
}
@Override
public void start() {
indexes.setSearchIndex(this);
indexes.addWriteIndex(this);
}
@Override
public void stop() {
openIndex.shutdown();
closedIndex.shutdown();
}
@Override
public Schema<ChangeData> getSchema() {
return schema;
}
@Override
public void close() {
stop();
}
@Override
public void replace(ChangeData cd) throws IOException {
String id = cd.getId().toString();
SolrInputDocument doc = toDocument(cd);
try {
if (cd.change().getStatus().isOpen()) {
closedIndex.deleteById(id);
openIndex.add(doc);
} else {
openIndex.deleteById(id);
closedIndex.add(doc);
}
} catch (OrmException | SolrServerException e) {
throw new IOException(e);
}
commit(openIndex);
commit(closedIndex);
}
@Override
public void delete(Change.Id id) throws IOException {
String idString = Integer.toString(id.get());
delete(idString, openIndex);
delete(idString, closedIndex);
}
private void delete(String id, CloudSolrServer index) throws IOException {
try {
index.deleteById(id);
commit(index);
} catch (SolrServerException e) {
throw new IOException(e);
}
}
@Override
public void deleteAll() throws IOException {
try {
openIndex.deleteByQuery("*:*");
closedIndex.deleteByQuery("*:*");
} catch (SolrServerException e) {
throw new IOException(e);
}
commit(openIndex);
commit(closedIndex);
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, int start, int limit)
throws QueryParseException {
Set<Change.Status> statuses = IndexRewriteImpl.getPossibleStatus(p);
List<SolrServer> indexes = Lists.newArrayListWithCapacity(2);
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
indexes.add(openIndex);
}
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(closedIndex);
}
return new QuerySource(indexes, queryBuilder.toQuery(p), start, limit,
getSorts());
}
private static List<SortClause> getSorts() {
return ImmutableList.of(
new SortClause(
ChangeField.UPDATED.getName(), SolrQuery.ORDER.desc),
new SortClause(
ChangeField.LEGACY_ID.getName(), SolrQuery.ORDER.desc));
}
private void commit(SolrServer server) throws IOException {
try {
server.commit();
} catch (SolrServerException e) {
throw new IOException(e);
}
}
private class QuerySource implements ChangeDataSource {
private final List<SolrServer> servers;
private final SolrQuery query;
public QuerySource(List<SolrServer> indexes, Query q, int start, int limit,
List<SortClause> sorts) {
this.servers = indexes;
query = new SolrQuery(q.toString());
query.setParam("shards.tolerant", true);
query.setParam("rows", Integer.toString(limit));
if (start != 0) {
query.setParam("start", Integer.toString(start));
}
query.setFields(ID_FIELD);
query.setSorts(sorts);
}
@Override
public int getCardinality() {
return 10; // TODO: estimate from solr?
}
@Override
public boolean hasChange() {
return false;
}
@Override
public String toString() {
return query.getQuery();
}
@Override
public ResultSet<ChangeData> read() throws OrmException {
try {
// TODO Sort documents during merge to select only top N.
SolrDocumentList docs = new SolrDocumentList();
for (SolrServer index : servers) {
docs.addAll(index.query(query).getResults());
}
List<ChangeData> result = Lists.newArrayListWithCapacity(docs.size());
for (SolrDocument doc : docs) {
Integer v = (Integer) doc.getFieldValue(ID_FIELD);
result.add(
changeDataFactory.create(db.get(), new Change.Id(v.intValue())));
}
final List<ChangeData> r = Collections.unmodifiableList(result);
return new ResultSet<ChangeData>() {
@Override
public Iterator<ChangeData> iterator() {
return r.iterator();
}
@Override
public List<ChangeData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (SolrServerException e) {
throw new OrmException(e);
}
}
}
private SolrInputDocument toDocument(ChangeData cd) {
SolrInputDocument result = new SolrInputDocument();
for (Values<ChangeData> values : schema.buildFields(cd, fillArgs)) {
add(result, values);
}
return result;
}
private void add(SolrInputDocument doc, Values<ChangeData> values) {
String name = values.getField().getName();
FieldType<?> type = values.getField().getType();
if (type == FieldType.INTEGER) {
for (Object value : values.getValues()) {
doc.addField(name, value);
}
} else if (type == FieldType.LONG) {
for (Object value : values.getValues()) {
doc.addField(name, value);
}
} else if (type == FieldType.TIMESTAMP) {
for (Object value : values.getValues()) {
doc.addField(name, ((Timestamp) value).getTime());
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX
|| type == FieldType.FULL_TEXT) {
for (Object value : values.getValues()) {
doc.addField(name, value);
}
} else {
throw FieldType.badFieldType(type);
}
}
@Override
public void markReady(boolean ready) throws IOException {
// TODO Move the schema version information to a special meta-document
FileBasedConfig cfg = new FileBasedConfig(
solrIndexConfig(sitePaths).toFile(),
FS.detect());
for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
cfg.setInt("index", e.getKey(), "schemaVersion",
ready ? e.getValue() : -1);
}
cfg.save();
}
}

78
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java
View File

@ -1,78 +0,0 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.solr;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Provider;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import org.eclipse.jgit.lib.Config;
import java.io.IOException;
public class SolrIndexModule extends LifecycleModule {
private final boolean checkVersion;
private final int threads;
private final String base;
public SolrIndexModule() {
this(true, 0, null);
}
public SolrIndexModule(boolean checkVersion, int threads, String base) {
this.checkVersion = checkVersion;
this.threads = threads;
this.base = base;
}
@Override
protected void configure() {
install(new IndexModule(threads));
bind(ChangeIndex.class).to(SolrChangeIndex.class);
listener().to(SolrChangeIndex.class);
if (checkVersion) {
listener().to(IndexVersionCheck.class);
}
}
@Provides
@Singleton
IndexConfig getIndexConfig(@GerritServerConfig Config cfg) {
return IndexConfig.fromConfig(cfg);
}
@Provides
@Singleton
public SolrChangeIndex getChangeIndex(@GerritServerConfig Config cfg,
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
SitePaths sitePaths,
IndexCollection indexes,
FillArgs fillArgs) throws IOException {
return new SolrChangeIndex(cfg, db, changeDataFactory, fillArgs, sitePaths,
indexes, ChangeSchemas.getLatest(), base);
}
}

1
gerrit-war/BUCK
View File

@ -17,7 +17,6 @@ java_library(
'//gerrit-reviewdb:server',
'//gerrit-server:server',
'//gerrit-server/src/main/prolog:common',
'//gerrit-solr:solr',
'//gerrit-sshd:sshd',
'//lib:guava',
'//lib:gwtorm',

4
gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java
View File

@ -61,7 +61,6 @@ import com.google.gerrit.server.schema.SchemaVersionCheck;
import com.google.gerrit.server.securestore.SecureStoreClassName;
import com.google.gerrit.server.ssh.NoSshModule;
import com.google.gerrit.server.ssh.SshAddressesModule;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.gerrit.sshd.SshHostKeyModule;
import com.google.gerrit.sshd.SshKeyCacheImpl;
import com.google.gerrit.sshd.SshModule;
@ -302,9 +301,6 @@ public class WebAppInitializer extends GuiceServletContextListener
case LUCENE:
changeIndexModule = new LuceneIndexModule();
break;
case SOLR:
changeIndexModule = new SolrIndexModule();
break;
default:
throw new IllegalStateException("unsupported index.type");
}

33
lib/solr/BUCK
View File

@ -1,33 +0,0 @@
include_defs('//lib/maven.defs')
# Java client library to use Solr over the network.
maven_jar(
name = 'solrj',
id = 'org.apache.solr:solr-solrj:4.3.1',
sha1 = '433fe37796e67eaeb4452f69eb1fae2de27cb7a8',
license = 'Apache2.0',
deps = [
':noggit',
':zookeeper',
'//lib/httpcomponents:httpclient',
'//lib/httpcomponents:httpmime',
'//lib/commons:io',
],
)
maven_jar(
name = 'noggit',
id = 'org.noggit:noggit:0.5',
sha1 = '8e6e65624d2e09a30190c6434abe23b7d4e5413c',
license = 'Apache2.0',
visibility = [],
)
maven_jar(
name = 'zookeeper',
id = 'org.apache.zookeeper:zookeeper:3.4.5',
sha1 = 'c0f69fb36526552a8f0bc548a6c33c49cf08e562',
license = 'Apache2.0',
deps = ['//lib/log:api'],
visibility = [],
)