diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneGroupIndex.java new file mode 100644 index 0000000000..436ee704f4 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneGroupIndex.java @@ -0,0 +1,198 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.lucene; + +import static com.google.gerrit.server.index.group.GroupField.UUID; + +import com.google.gerrit.reviewdb.client.AccountGroup; +import com.google.gerrit.server.account.GroupCache; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.QueryOptions; +import com.google.gerrit.server.index.Schema; +import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.query.DataSource; +import com.google.gerrit.server.query.Predicate; +import com.google.gerrit.server.query.QueryParseException; +import com.google.gwtorm.server.OrmException; +import com.google.gwtorm.server.ResultSet; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; +import org.eclipse.jgit.lib.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class LuceneGroupIndex extends + AbstractLuceneIndex implements GroupIndex { + private static final Logger log = + LoggerFactory.getLogger(LuceneGroupIndex.class); + + private static final String GROUPS = "groups"; + + private static final String UUID_SORT_FIELD = sortFieldName(UUID); + + private static Term idTerm(AccountGroup group) { + return idTerm(group.getGroupUUID()); + } + + private static Term idTerm(AccountGroup.UUID uuid) { + return QueryBuilder.stringTerm(UUID.getName(), uuid.get()); + } + + private final GerritIndexWriterConfig indexWriterConfig; + private final QueryBuilder queryBuilder; + private final GroupCache groupCache; + + private static Directory dir(Schema schema, Config cfg, + SitePaths sitePaths) throws IOException { + if (LuceneIndexModule.isInMemoryTest(cfg)) { + return new RAMDirectory(); + } + Path indexDir = + LuceneVersionManager.getDir(sitePaths, GROUPS + "_", schema); + return FSDirectory.open(indexDir); + } + + @Inject + LuceneGroupIndex( + @GerritServerConfig Config cfg, + SitePaths sitePaths, + GroupCache groupCache, + @Assisted Schema schema) throws IOException { + super(schema, sitePaths, dir(schema, cfg, sitePaths), GROUPS, null, + new GerritIndexWriterConfig(cfg, GROUPS), new SearcherFactory()); + this.groupCache = groupCache; + + indexWriterConfig = + new GerritIndexWriterConfig(cfg, GROUPS); + queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer()); + } + + @Override + public void replace(AccountGroup group) throws IOException { + try { + // No parts of FillArgs are currently required, just use null. + replace(idTerm(group), toDocument(group, null)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new IOException(e); + } + } + + @Override + public void delete(AccountGroup.UUID key) throws IOException { + try { + delete(idTerm(key)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new IOException(e); + } + } + + @Override + public DataSource getSource(Predicate p, + QueryOptions opts) throws QueryParseException { + return new QuerySource(opts, queryBuilder.toQuery(p), + new Sort(new SortField(UUID_SORT_FIELD, SortField.Type.STRING, false))); + } + + private class QuerySource implements DataSource { + private final QueryOptions opts; + private final Query query; + private final Sort sort; + + private QuerySource(QueryOptions opts, Query query, Sort sort) { + this.opts = opts; + this.query = query; + this.sort = sort; + } + + @Override + public int getCardinality() { + return 10; + } + + @Override + public ResultSet read() throws OrmException { + IndexSearcher searcher = null; + try { + searcher = acquire(); + int realLimit = opts.start() + opts.limit(); + TopFieldDocs docs = searcher.search(query, realLimit, sort); + List result = new ArrayList<>(docs.scoreDocs.length); + for (int i = opts.start(); i < docs.scoreDocs.length; i++) { + ScoreDoc sd = docs.scoreDocs[i]; + Document doc = searcher.doc(sd.doc, IndexUtils.groupFields(opts)); + result.add(toAccountGroup(doc)); + } + final List r = Collections.unmodifiableList(result); + return new ResultSet() { + @Override + public Iterator iterator() { + return r.iterator(); + } + + @Override + public List toList() { + return r; + } + + @Override + public void close() { + // Do nothing. + } + }; + } catch (IOException e) { + throw new OrmException(e); + } finally { + if (searcher != null) { + try { + release(searcher); + } catch (IOException e) { + log.warn("cannot release Lucene searcher", e); + } + } + } + } + } + + private AccountGroup toAccountGroup(Document doc) { + AccountGroup.UUID uuid = + new AccountGroup.UUID(doc.getField(UUID.getName()).stringValue()); + // Use the GroupCache rather than depending on any stored fields in the + // document (of which there shouldn't be any). + return groupCache.get(uuid); + } +} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java index d23c5bb2bc..1d1ca95f24 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -22,6 +22,7 @@ import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.SingleVersionModule; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; +import com.google.gerrit.server.index.group.GroupIndex; import com.google.inject.Provides; import com.google.inject.Singleton; import com.google.inject.assistedinject.FactoryModuleBuilder; @@ -59,14 +60,18 @@ public class LuceneIndexModule extends LifecycleModule { @Override protected void configure() { + install( + new FactoryModuleBuilder() + .implement(AccountIndex.class, LuceneAccountIndex.class) + .build(AccountIndex.Factory.class)); install( new FactoryModuleBuilder() .implement(ChangeIndex.class, LuceneChangeIndex.class) .build(ChangeIndex.Factory.class)); install( new FactoryModuleBuilder() - .implement(AccountIndex.class, LuceneAccountIndex.class) - .build(AccountIndex.Factory.class)); + .implement(GroupIndex.class, LuceneGroupIndex.class) + .build(GroupIndex.Factory.class)); install(new IndexModule(threads)); if (singleVersions == null) { diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java index a993b49478..cfec648f81 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java @@ -15,6 +15,7 @@ package com.google.gerrit.lucene; import static com.google.common.base.Preconditions.checkArgument; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.lucene.search.BooleanClause.Occur.MUST; import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT; import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; @@ -54,6 +55,12 @@ public class QueryBuilder { return new Term(name, builder.get()); } + static Term stringTerm(String name, String value) { + BytesRefBuilder builder = new BytesRefBuilder(); + builder.append(value.getBytes(UTF_8), 0, value.length()); + return new Term(name, builder.get()); + } + private final Schema schema; private final org.apache.lucene.util.QueryBuilder queryBuilder; diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java index ed196c1739..1706761e56 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java @@ -14,10 +14,12 @@ package com.google.gerrit.server.index; +import com.google.gerrit.reviewdb.client.AccountGroup; import com.google.gerrit.server.account.AccountState; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.change.DummyChangeIndex; +import com.google.gerrit.server.index.group.GroupIndex; import com.google.gerrit.server.query.change.ChangeData; import com.google.inject.AbstractModule; @@ -36,6 +38,13 @@ public class DummyIndexModule extends AbstractModule { } } + private static class DummyGroupIndexFactory implements GroupIndex.Factory { + @Override + public GroupIndex create(Schema schema) { + throw new UnsupportedOperationException(); + } + } + @Override protected void configure() { install(new IndexModule(1)); @@ -43,5 +52,6 @@ public class DummyIndexModule extends AbstractModule { bind(Index.class).toInstance(new DummyChangeIndex()); bind(AccountIndex.Factory.class).toInstance(new DummyAccountIndexFactory()); bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory()); + bind(GroupIndex.Factory.class).toInstance(new DummyGroupIndexFactory()); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java index 9e0be86712..13e7b85582 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java @@ -36,6 +36,8 @@ import com.google.gerrit.server.index.change.ChangeIndexDefinition; import com.google.gerrit.server.index.change.ChangeIndexRewriter; import com.google.gerrit.server.index.change.ChangeIndexer; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; +import com.google.gerrit.server.index.group.GroupIndexDefinition; +import com.google.gerrit.server.index.group.GroupSchemaDefinitions; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Provides; @@ -61,7 +63,8 @@ public class IndexModule extends LifecycleModule { public static final ImmutableCollection> ALL_SCHEMA_DEFS = ImmutableList.> of( AccountSchemaDefinitions.INSTANCE, - ChangeSchemaDefinitions.INSTANCE); + ChangeSchemaDefinitions.INSTANCE, + GroupSchemaDefinitions.INSTANCE); /** Type of secondary index. */ public static IndexType getIndexType(Injector injector) { @@ -103,10 +106,12 @@ public class IndexModule extends LifecycleModule { @Provides Collection> getIndexDefinitions( AccountIndexDefinition accounts, - ChangeIndexDefinition changes) { + ChangeIndexDefinition changes, + GroupIndexDefinition groups) { Collection> result = ImmutableList.> of( accounts, + groups, changes); Set expected = FluentIterable.from(ALL_SCHEMA_DEFS) .transform(SchemaDefinitions::getName) diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java index bcc7f7b527..eceec59dc0 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java @@ -14,7 +14,6 @@ package com.google.gerrit.server.index; -import static com.google.gerrit.server.index.account.AccountField.ID; import static com.google.gerrit.server.index.change.ChangeField.CHANGE; import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID; import static com.google.gerrit.server.index.change.ChangeField.PROJECT; @@ -23,6 +22,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.account.AccountField; +import com.google.gerrit.server.index.group.GroupField; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -47,9 +48,9 @@ public final class IndexUtils { public static Set accountFields(QueryOptions opts) { Set fs = opts.fields(); - return fs.contains(ID.getName()) + return fs.contains(AccountField.ID.getName()) ? fs - : Sets.union(fs, ImmutableSet.of(ID.getName())); + : Sets.union(fs, ImmutableSet.of(AccountField.ID.getName())); } public static Set changeFields(QueryOptions opts) { @@ -68,6 +69,13 @@ public final class IndexUtils { ImmutableSet.of(LEGACY_ID.getName(), PROJECT.getName())); } + public static Set groupFields(QueryOptions opts) { + Set fs = opts.fields(); + return fs.contains(GroupField.UUID.getName()) + ? fs + : Sets.union(fs, ImmutableSet.of(GroupField.UUID.getName())); + } + private IndexUtils() { // hide default constructor }