Merge changes from topic 'account-index'

* changes:
  Make LimitPredicate generic
  Add AccountIndexRewriter
  Add IndexedAccountQuery
  Reindex account whenever account is evicted from cache
  Reindex: Allow to specify index name for reindex operation
  Reindex: Add --list option to list available indices
  Initial implementation of account index in Lucene
This commit is contained in:
David Pursehouse
2016-06-24 05:28:16 +00:00
committed by Gerrit Code Review
48 changed files with 731 additions and 256 deletions

View File

@@ -16,6 +16,8 @@ package com.google.gerrit.server.account;
import com.google.gerrit.reviewdb.client.Account;
import java.io.IOException;
/** Caches important (but small) account state to avoid database hits. */
public interface AccountCache {
AccountState get(Account.Id accountId);
@@ -24,7 +26,7 @@ public interface AccountCache {
AccountState getByUsername(String username);
void evict(Account.Id accountId);
void evict(Account.Id accountId) throws IOException;
void evictByUsername(String username);
}

View File

@@ -26,6 +26,8 @@ import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.reviewdb.client.AccountGroupMember;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.account.AccountIndexCollection;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
@@ -74,12 +76,15 @@ public class AccountCacheImpl implements AccountCache {
private final LoadingCache<Account.Id, AccountState> byId;
private final LoadingCache<String, Optional<Account.Id>> byName;
private final AccountIndexCollection indexes;
@Inject
AccountCacheImpl(@Named(BYID_NAME) LoadingCache<Account.Id, AccountState> byId,
@Named(BYUSER_NAME) LoadingCache<String, Optional<Account.Id>> byUsername) {
@Named(BYUSER_NAME) LoadingCache<String, Optional<Account.Id>> byUsername,
AccountIndexCollection indexes) {
this.byId = byId;
this.byName = byUsername;
this.indexes = indexes;
}
@Override
@@ -109,9 +114,16 @@ public class AccountCacheImpl implements AccountCache {
}
@Override
public void evict(Account.Id accountId) {
public void evict(Account.Id accountId) throws IOException {
if (accountId != null) {
byId.invalidate(accountId);
index(accountId);
}
}
private void index(Account.Id id) throws IOException {
for (Index<?, AccountState> i : indexes.getWriteIndexes()) {
i.replace(get(id));
}
}

View File

@@ -37,6 +37,7 @@ import com.google.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -101,7 +102,8 @@ public class AccountManager {
* @throws AccountException the account does not exist, and cannot be created,
* or exists, but cannot be located, or is inactive.
*/
public AuthResult authenticate(AuthRequest who) throws AccountException {
public AuthResult authenticate(AuthRequest who)
throws AccountException, IOException {
who = realm.authenticate(who);
try {
try (ReviewDb db = schema.open()) {
@@ -152,7 +154,8 @@ public class AccountManager {
}
private void update(ReviewDb db, AuthRequest who, AccountExternalId extId)
throws OrmException, NameAlreadyUsedException, InvalidUserNameException {
throws OrmException, NameAlreadyUsedException, InvalidUserNameException,
IOException {
IdentifiedUser user = userFactory.create(extId.getAccountId());
Account toUpdate = null;
@@ -214,7 +217,7 @@ public class AccountManager {
}
private AuthResult create(ReviewDb db, AuthRequest who)
throws OrmException, AccountException {
throws OrmException, AccountException, IOException {
Account.Id newId = new Account.Id(db.nextAccountId());
Account account = new Account(newId, TimeUtil.nowTs());
AccountExternalId extId = createId(newId, who);
@@ -340,7 +343,7 @@ public class AccountManager {
* cannot be linked at this time.
*/
public AuthResult link(Account.Id to, AuthRequest who)
throws AccountException, OrmException {
throws AccountException, OrmException, IOException {
try (ReviewDb db = schema.open()) {
AccountExternalId.Key key = id(who);
AccountExternalId extId = getAccountExternalId(db, key);
@@ -392,7 +395,7 @@ public class AccountManager {
* cannot be linked at this time.
*/
public AuthResult updateLink(Account.Id to, AuthRequest who) throws OrmException,
AccountException {
AccountException, IOException {
try (ReviewDb db = schema.open()) {
AccountExternalId.Key key = id(who);
List<AccountExternalId.Key> filteredKeysByScheme =
@@ -429,7 +432,7 @@ public class AccountManager {
* cannot be unlinked at this time.
*/
public AuthResult unlink(Account.Id from, AuthRequest who)
throws AccountException, OrmException {
throws AccountException, OrmException, IOException {
try (ReviewDb db = schema.open()) {
AccountExternalId.Key key = id(who);
AccountExternalId extId = getAccountExternalId(db, key);

View File

@@ -29,6 +29,7 @@ import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -71,7 +72,7 @@ public class ChangeUserName implements Callable<VoidResult> {
@Override
public VoidResult call() throws OrmException, NameAlreadyUsedException,
InvalidUserNameException {
InvalidUserNameException, IOException {
final Collection<AccountExternalId> old = old();
if (!old.isEmpty()) {
throw new IllegalStateException(USERNAME_CANNOT_BE_CHANGED);

View File

@@ -39,6 +39,8 @@ import org.apache.commons.validator.routines.EmailValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class CreateEmail implements RestModifyView<AccountResource, EmailInput> {
private static final Logger log = LoggerFactory.getLogger(CreateEmail.class);
@@ -75,7 +77,7 @@ public class CreateEmail implements RestModifyView<AccountResource, EmailInput>
public Response<EmailInfo> apply(AccountResource rsrc, EmailInput input)
throws AuthException, BadRequestException, ResourceConflictException,
ResourceNotFoundException, OrmException, EmailException,
MethodNotAllowedException {
MethodNotAllowedException, IOException {
if (self.get() != rsrc.getUser()
&& !self.get().getCapabilities().canModifyAccount()) {
throw new AuthException("not allowed to add email address");
@@ -104,7 +106,7 @@ public class CreateEmail implements RestModifyView<AccountResource, EmailInput>
public Response<EmailInfo> apply(IdentifiedUser user, EmailInput input)
throws AuthException, BadRequestException, ResourceConflictException,
ResourceNotFoundException, OrmException, EmailException,
MethodNotAllowedException {
MethodNotAllowedException, IOException {
if (input.email != null && !email.equals(input.email)) {
throw new BadRequestException("email address must match URL");
}

View File

@@ -27,6 +27,7 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.Collections;
@RequiresCapability(GlobalCapability.MODIFY_ACCOUNT)
@@ -46,7 +47,7 @@ public class DeleteActive implements RestModifyView<AccountResource, Input> {
@Override
public Response<?> apply(AccountResource rsrc, Input input)
throws ResourceNotFoundException, OrmException {
throws ResourceNotFoundException, OrmException, IOException {
Account a = dbProvider.get().accounts().get(rsrc.getUser().getAccountId());
if (a == null) {
throw new ResourceNotFoundException("account not found");

View File

@@ -31,6 +31,8 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
@Singleton
public class DeleteEmail implements RestModifyView<AccountResource.Email, Input> {
public static class Input {
@@ -53,7 +55,8 @@ public class DeleteEmail implements RestModifyView<AccountResource.Email, Input>
@Override
public Response<?> apply(AccountResource.Email rsrc, Input input)
throws AuthException, ResourceNotFoundException,
ResourceConflictException, MethodNotAllowedException, OrmException {
ResourceConflictException, MethodNotAllowedException, OrmException,
IOException {
if (self.get() != rsrc.getUser()
&& !self.get().getCapabilities().canModifyAccount()) {
throw new AuthException("not allowed to delete email address");
@@ -63,7 +66,7 @@ public class DeleteEmail implements RestModifyView<AccountResource.Email, Input>
public Response<?> apply(IdentifiedUser user, String email)
throws ResourceNotFoundException, ResourceConflictException,
MethodNotAllowedException, OrmException {
MethodNotAllowedException, OrmException, IOException {
if (!realm.allowsEdit(FieldName.REGISTER_NEW_EMAIL)) {
throw new MethodNotAllowedException("realm does not allow deleting emails");
}

View File

@@ -27,6 +27,7 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.Collections;
@RequiresCapability(GlobalCapability.MODIFY_ACCOUNT)
@@ -46,7 +47,7 @@ public class PutActive implements RestModifyView<AccountResource, Input> {
@Override
public Response<String> apply(AccountResource rsrc, Input input)
throws ResourceNotFoundException, OrmException {
throws ResourceNotFoundException, OrmException, IOException {
Account a = dbProvider.get().accounts().get(rsrc.getUser().getAccountId());
if (a == null) {
throw new ResourceNotFoundException("account not found");

View File

@@ -34,6 +34,7 @@ import com.google.inject.Singleton;
import org.apache.commons.codec.binary.Base64;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Collections;
@@ -69,8 +70,9 @@ public class PutHttpPassword implements RestModifyView<AccountResource, Input> {
}
@Override
public Response<String> apply(AccountResource rsrc, Input input) throws AuthException,
ResourceNotFoundException, ResourceConflictException, OrmException {
public Response<String> apply(AccountResource rsrc, Input input)
throws AuthException, ResourceNotFoundException,
ResourceConflictException, OrmException, IOException {
if (input == null) {
input = new Input();
}
@@ -101,7 +103,8 @@ public class PutHttpPassword implements RestModifyView<AccountResource, Input> {
}
public Response<String> apply(IdentifiedUser user, String newPassword)
throws ResourceNotFoundException, ResourceConflictException, OrmException {
throws ResourceNotFoundException, ResourceConflictException, OrmException,
IOException {
if (user.getUserName() == null) {
throw new ResourceConflictException("username must be set");
}

View File

@@ -36,6 +36,7 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.Collections;
@Singleton
@@ -62,7 +63,7 @@ public class PutName implements RestModifyView<AccountResource, Input> {
@Override
public Response<String> apply(AccountResource rsrc, Input input)
throws AuthException, MethodNotAllowedException,
ResourceNotFoundException, OrmException {
ResourceNotFoundException, OrmException, IOException {
if (self.get() != rsrc.getUser()
&& !self.get().getCapabilities().canModifyAccount()) {
throw new AuthException("not allowed to change name");
@@ -71,7 +72,8 @@ public class PutName implements RestModifyView<AccountResource, Input> {
}
public Response<String> apply(IdentifiedUser user, Input input)
throws MethodNotAllowedException, ResourceNotFoundException, OrmException {
throws MethodNotAllowedException, ResourceNotFoundException, OrmException,
IOException {
if (input == null) {
input = new Input();
}

View File

@@ -28,6 +28,7 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.Collections;
@Singleton
@@ -50,7 +51,8 @@ public class PutPreferred implements
@Override
public Response<String> apply(AccountResource.Email rsrc, Input input)
throws AuthException, ResourceNotFoundException, OrmException {
throws AuthException, ResourceNotFoundException, OrmException,
IOException {
if (self.get() != rsrc.getUser()
&& !self.get().getCapabilities().canModifyAccount()) {
throw new AuthException("not allowed to set preferred email address");
@@ -59,7 +61,7 @@ public class PutPreferred implements
}
public Response<String> apply(IdentifiedUser user, String email)
throws ResourceNotFoundException, OrmException {
throws ResourceNotFoundException, OrmException, IOException {
Account a = dbProvider.get().accounts().get(user.getAccountId());
if (a == null) {
throw new ResourceNotFoundException("account not found");

View File

@@ -30,6 +30,8 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
@Singleton
public class PutUsername implements RestModifyView<AccountResource, Input> {
public static class Input {
@@ -56,7 +58,7 @@ public class PutUsername implements RestModifyView<AccountResource, Input> {
@Override
public String apply(AccountResource rsrc, Input input) throws AuthException,
MethodNotAllowedException, UnprocessableEntityException,
ResourceConflictException, OrmException {
ResourceConflictException, OrmException, IOException {
if (self.get() != rsrc.getUser()
&& !self.get().getCapabilities().canAdministrateServer()) {
throw new AuthException("not allowed to set username");

View File

@@ -311,7 +311,7 @@ public class AccountApiImpl implements AccountApi {
new AccountResource.Email(account.getUser(), input.email);
try {
createEmailFactory.create(input.email).apply(rsrc, input);
} catch (EmailException | OrmException e) {
} catch (EmailException | OrmException | IOException e) {
throw new RestApiException("Cannot add email", e);
}
}

View File

@@ -44,6 +44,7 @@ import com.google.gwtorm.server.OrmException;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@@ -217,7 +218,7 @@ class GroupApiImpl implements GroupApi {
try {
addMembers.apply(
rsrc, AddMembers.Input.fromMembers(Arrays.asList(members)));
} catch (OrmException e) {
} catch (OrmException | IOException e) {
throw new RestApiException("Cannot add group members", e);
}
}
@@ -227,7 +228,7 @@ class GroupApiImpl implements GroupApi {
try {
deleteMembers.apply(
rsrc, AddMembers.Input.fromMembers(Arrays.asList(members)));
} catch (OrmException e) {
} catch (OrmException | IOException e) {
throw new RestApiException("Cannot remove group members", e);
}
}

View File

@@ -90,7 +90,7 @@ class GroupsImpl implements Groups {
GroupInfo info = createGroup.create(in.name)
.apply(TopLevelResource.INSTANCE, in);
return id(info.id);
} catch (OrmException e) {
} catch (OrmException | IOException e) {
throw new RestApiException("Cannot create group " + in.name, e);
}
}

View File

@@ -32,6 +32,8 @@ import org.kohsuke.args4j.spi.OptionHandler;
import org.kohsuke.args4j.spi.Parameters;
import org.kohsuke.args4j.spi.Setter;
import java.io.IOException;
public class AccountIdHandler extends OptionHandler<Account.Id> {
private final AccountResolver accountResolver;
private final AccountManager accountManager;
@@ -76,7 +78,7 @@ public class AccountIdHandler extends OptionHandler<Account.Id> {
throw new CmdLineException(owner, "user \"" + token + "\" not found");
}
}
} catch (OrmException e) {
} catch (OrmException | IOException e) {
throw new CmdLineException(owner, "database is down");
}
setter.addValue(accountId);
@@ -84,7 +86,7 @@ public class AccountIdHandler extends OptionHandler<Account.Id> {
}
private Account.Id createAccountByLdap(String user)
throws CmdLineException {
throws CmdLineException, IOException {
if (!user.matches(Account.USER_NAME_PATTERN)) {
throw new CmdLineException(owner, "user \"" + user + "\" not found");
}

View File

@@ -30,6 +30,8 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
@Singleton
public class ConfirmEmail implements RestModifyView<ConfigResource, Input> {
public static class Input {
@@ -53,7 +55,7 @@ public class ConfirmEmail implements RestModifyView<ConfigResource, Input> {
@Override
public Response<?> apply(ConfigResource rsrc, Input input)
throws AuthException, UnprocessableEntityException, AccountException,
OrmException {
OrmException, IOException {
CurrentUser user = self.get();
if (!user.isIdentifiedUser()) {
throw new AuthException("Authentication required");

View File

@@ -45,6 +45,7 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -114,7 +115,7 @@ public class AddMembers implements RestModifyView<GroupResource, Input> {
@Override
public List<AccountInfo> apply(GroupResource resource, Input input)
throws AuthException, MethodNotAllowedException,
UnprocessableEntityException, OrmException {
UnprocessableEntityException, OrmException, IOException {
AccountGroup internalGroup = resource.toAccountGroup();
if (internalGroup == null) {
throw new MethodNotAllowedException();
@@ -142,7 +143,7 @@ public class AddMembers implements RestModifyView<GroupResource, Input> {
}
private Account findAccount(String nameOrEmail) throws AuthException,
UnprocessableEntityException, OrmException {
UnprocessableEntityException, OrmException, IOException {
try {
return accounts.parse(nameOrEmail).getAccount();
} catch (UnprocessableEntityException e) {
@@ -174,7 +175,8 @@ public class AddMembers implements RestModifyView<GroupResource, Input> {
}
public void addMembers(AccountGroup.Id groupId,
Collection<? extends Account.Id> newMemberIds) throws OrmException {
Collection<? extends Account.Id> newMemberIds)
throws OrmException, IOException {
Map<Account.Id, AccountGroupMember> newAccountGroupMembers = new HashMap<>();
for (Account.Id accId : newMemberIds) {
if (!newAccountGroupMembers.containsKey(accId)) {
@@ -197,7 +199,7 @@ public class AddMembers implements RestModifyView<GroupResource, Input> {
}
}
private Account createAccountByLdap(String user) {
private Account createAccountByLdap(String user) throws IOException {
if (!user.matches(Account.USER_NAME_PATTERN)) {
return null;
}
@@ -238,7 +240,7 @@ public class AddMembers implements RestModifyView<GroupResource, Input> {
@Override
public AccountInfo apply(GroupResource resource, PutMember.Input input)
throws AuthException, MethodNotAllowedException,
ResourceNotFoundException, OrmException {
ResourceNotFoundException, OrmException, IOException {
AddMembers.Input in = new AddMembers.Input();
in._oneMember = id;
try {

View File

@@ -50,6 +50,7 @@ import com.google.inject.assistedinject.Assisted;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.PersonIdent;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
@@ -98,7 +99,7 @@ public class CreateGroup implements RestModifyView<TopLevelResource, GroupInput>
@Override
public GroupInfo apply(TopLevelResource resource, GroupInput input)
throws BadRequestException, UnprocessableEntityException,
ResourceConflictException, OrmException {
ResourceConflictException, OrmException, IOException {
if (input == null) {
input = new GroupInput();
}
@@ -138,7 +139,7 @@ public class CreateGroup implements RestModifyView<TopLevelResource, GroupInput>
}
private AccountGroup createGroup(CreateGroupArgs createGroupArgs)
throws OrmException, ResourceConflictException {
throws OrmException, ResourceConflictException, IOException {
// Do not allow creating groups with the same name as system groups
List<String> sysGroupNames = SystemGroupBackend.getNames();

View File

@@ -34,6 +34,7 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -62,7 +63,7 @@ public class DeleteMembers implements RestModifyView<GroupResource, Input> {
@Override
public Response<?> apply(GroupResource resource, Input input)
throws AuthException, MethodNotAllowedException,
UnprocessableEntityException, OrmException {
UnprocessableEntityException, OrmException, IOException {
AccountGroup internalGroup = resource.toAccountGroup();
if (internalGroup == null) {
throw new MethodNotAllowedException();
@@ -125,7 +126,7 @@ public class DeleteMembers implements RestModifyView<GroupResource, Input> {
@Override
public Response<?> apply(MemberResource resource, Input input)
throws AuthException, MethodNotAllowedException,
UnprocessableEntityException, OrmException {
UnprocessableEntityException, OrmException, IOException {
AddMembers.Input in = new AddMembers.Input();
in._oneMember = resource.getMember().getAccountId().toString();
return delete.get().apply(resource, in);

View File

@@ -14,6 +14,8 @@
package com.google.gerrit.server.index;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.DummyChangeIndex;
import com.google.gerrit.server.query.change.ChangeData;
@@ -27,11 +29,19 @@ public class DummyIndexModule extends AbstractModule {
}
}
private static class DummyAccountIndexFactory implements AccountIndex.Factory {
@Override
public AccountIndex create(Schema<AccountState> schema) {
throw new UnsupportedOperationException();
}
}
@Override
protected void configure() {
install(new IndexModule(1));
bind(IndexConfig.class).toInstance(IndexConfig.createDefault());
bind(Index.class).toInstance(new DummyChangeIndex());
bind(AccountIndex.Factory.class).toInstance(new DummyAccountIndexFactory());
bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory());
}
}

View File

@@ -26,11 +26,15 @@ import com.google.common.util.concurrent.MoreExecutors;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.index.account.AccountIndexCollection;
import com.google.gerrit.server.index.account.AccountIndexDefinition;
import com.google.gerrit.server.index.account.AccountIndexRewriter;
import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeIndexDefinition;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.index.change.IndexRewriter;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provides;
@@ -55,6 +59,7 @@ public class IndexModule extends LifecycleModule {
public static final ImmutableCollection<SchemaDefinitions<?>> ALL_SCHEMA_DEFS =
ImmutableList.<SchemaDefinitions<?>> of(
AccountSchemaDefinitions.INSTANCE,
ChangeSchemaDefinitions.INSTANCE);
/** Type of secondary index. */
@@ -83,7 +88,11 @@ public class IndexModule extends LifecycleModule {
@Override
protected void configure() {
bind(IndexRewriter.class);
bind(AccountIndexRewriter.class);
bind(AccountIndexCollection.class);
listener().to(AccountIndexCollection.class);
bind(ChangeIndexRewriter.class);
bind(ChangeIndexCollection.class);
listener().to(ChangeIndexCollection.class);
factory(ChangeIndexer.Factory.class);
@@ -91,9 +100,12 @@ public class IndexModule extends LifecycleModule {
@Provides
Collection<IndexDefinition<?, ?, ?>> getIndexDefinitions(
AccountIndexDefinition accounts,
ChangeIndexDefinition changes) {
Collection<IndexDefinition<?, ?, ?>> result =
ImmutableList.<IndexDefinition<?, ?, ?>> of(changes);
ImmutableList.<IndexDefinition<?, ?, ?>> of(
accounts,
changes);
Set<String> expected = FluentIterable.from(ALL_SCHEMA_DEFS)
.transform(new Function<SchemaDefinitions<?>, String>() {
@Override

View File

@@ -0,0 +1,24 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
public interface IndexRewriter<T> {
Predicate<T> rewrite(Predicate<T> in, QueryOptions opts)
throws QueryParseException;
}

View File

@@ -0,0 +1,183 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index;
import com.google.common.base.Function;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.gerrit.server.query.DataSource;
import com.google.gerrit.server.query.Paginated;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* Wrapper combining an {@link IndexPredicate} together with a
* {@link DataSource} that returns matching results from the index.
* <p>
* Appropriate to return as the rootmost predicate that can be processed using
* the secondary index; such predicates must also implement {@link DataSource}
* to be chosen by the query processor.
*
* @param <I> The type of the IDs by which the entities are stored in the index.
* @param <T> The type of the entities that are stored in the index.
*/
public class IndexedQuery<I, T> extends Predicate<T>
implements DataSource<T>, Paginated<T> {
protected final Index<I, T> index;
private QueryOptions opts;
private final Predicate<T> pred;
private DataSource<T> source;
private final Map<T, DataSource<T>> fromSource;
public IndexedQuery(Index<I, T> index, Predicate<T> pred,
QueryOptions opts) throws QueryParseException {
this.index = index;
this.opts = opts;
this.pred = pred;
this.source = index.getSource(pred, this.opts);
this.fromSource = new HashMap<>();
}
@Override
public int getChildCount() {
return 1;
}
@Override
public Predicate<T> getChild(int i) {
if (i == 0) {
return pred;
}
throw new ArrayIndexOutOfBoundsException(i);
}
@Override
public List<Predicate<T>> getChildren() {
return ImmutableList.of(pred);
}
@Override
public QueryOptions getOptions() {
return opts;
}
@Override
public int getCardinality() {
return source != null ? source.getCardinality() : opts.limit();
}
@Override
public ResultSet<T> read() throws OrmException {
final DataSource<T> currSource = source;
final ResultSet<T> rs = currSource.read();
return new ResultSet<T>() {
@Override
public Iterator<T> iterator() {
return Iterables.transform(
rs,
new Function<T, T>() {
@Override
public
T apply(T t) {
fromSource.put(t, currSource);
return t;
}
}).iterator();
}
@Override
public List<T> toList() {
List<T> r = rs.toList();
for (T t : r) {
fromSource.put(t, currSource);
}
return r;
}
@Override
public void close() {
rs.close();
}
};
}
@Override
public ResultSet<T> restart(int start) throws OrmException {
opts = opts.withStart(start);
try {
source = index.getSource(pred, opts);
} catch (QueryParseException e) {
// Don't need to show this exception to the user; the only thing that
// changed about pred was its start, and any other QPEs that might happen
// should have already thrown from the constructor.
throw new OrmException(e);
}
// Don't convert start to a limit, since the caller of this method (see
// AndSource) has calculated the actual number to skip.
return read();
}
@Override
public Predicate<T> copy(Collection<? extends Predicate<T>> children) {
return this;
}
@Override
public boolean match(T t) throws OrmException {
return (source != null && fromSource.get(t) == source) || pred.match(t);
}
@Override
public int getCost() {
// Index queries are assumed to be cheaper than any other type of query, so
// so try to make sure they get picked. Note that pred's cost may be higher
// because it doesn't know whether it's being used in an index query or not.
return 1;
}
@Override
public int hashCode() {
return pred.hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null || getClass() != other.getClass()) {
return false;
}
IndexedQuery<?, ?> o = (IndexedQuery<?, ?>) other;
return pred.equals(o.pred)
&& opts.equals(o.opts);
}
@Override
public String toString() {
return MoreObjects.toStringHelper("index")
.add("p", pred)
.add("opts", opts)
.toString();
}
}

View File

@@ -0,0 +1,44 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.account;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.IndexRewriter;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@Singleton
public class AccountIndexRewriter implements IndexRewriter<AccountState> {
private final AccountIndexCollection indexes;
@Inject
AccountIndexRewriter(AccountIndexCollection indexes) {
this.indexes = indexes;
}
@Override
public Predicate<AccountState> rewrite(Predicate<AccountState> in,
QueryOptions opts) throws QueryParseException {
AccountIndex index = indexes.getSearchIndex();
checkNotNull(index, "no active search index configured for accounts");
return new IndexedAccountQuery(index, in, opts);
}
}

View File

@@ -0,0 +1,34 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.account;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.IndexedQuery;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.query.DataSource;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
public class IndexedAccountQuery extends IndexedQuery<Account.Id, AccountState>
implements DataSource<AccountState> {
public IndexedAccountQuery(Index<Account.Id, AccountState> index,
Predicate<AccountState> pred, QueryOptions opts)
throws QueryParseException {
super(index, pred, opts);
}
}

View File

@@ -20,16 +20,18 @@ import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Change.Status;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.IndexRewriter;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.query.AndPredicate;
import com.google.gerrit.server.query.LimitPredicate;
import com.google.gerrit.server.query.NotPredicate;
import com.google.gerrit.server.query.OrPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.AndSource;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
import com.google.gerrit.server.query.change.ChangeStatusPredicate;
import com.google.gerrit.server.query.change.LimitPredicate;
import com.google.gerrit.server.query.change.OrSource;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -43,7 +45,7 @@ import java.util.Set;
/** Rewriter that pushes boolean logic into the secondary index. */
@Singleton
public class IndexRewriter {
public class ChangeIndexRewriter implements IndexRewriter<ChangeData> {
/** Set of all open change statuses. */
public static final Set<Change.Status> OPEN_STATUSES;
@@ -124,12 +126,13 @@ public class IndexRewriter {
private final IndexConfig config;
@Inject
IndexRewriter(ChangeIndexCollection indexes,
ChangeIndexRewriter(ChangeIndexCollection indexes,
IndexConfig config) {
this.indexes = indexes;
this.config = config;
}
@Override
public Predicate<ChangeData> rewrite(Predicate<ChangeData> in,
QueryOptions opts) throws QueryParseException {
ChangeIndex index = indexes.getSearchIndex();
@@ -172,7 +175,7 @@ public class IndexRewriter {
// Replace any limits with the limit provided by the caller. The caller
// should have already searched the predicate tree for limit predicates
// and included that in their limit computation.
return new LimitPredicate(opts.limit());
return new LimitPredicate<>(ChangeQueryBuilder.FIELD_LIMIT, opts.limit());
} else if (!isRewritePossible(in)) {
return null; // magic to indicate "in" cannot be rewritten
}

View File

@@ -18,28 +18,19 @@ import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.primitives.Ints;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.IndexedQuery;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.query.DataSource;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gerrit.server.query.change.Paginated;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/**
@@ -50,8 +41,8 @@ import java.util.Set;
* the secondary index; such predicates must also implement
* {@link ChangeDataSource} to be chosen by the query processor.
*/
public class IndexedChangeQuery extends Predicate<ChangeData>
implements ChangeDataSource, Paginated {
public class IndexedChangeQuery extends IndexedQuery<Change.Id, ChangeData>
implements ChangeDataSource {
public static QueryOptions oneResult() {
return createOptions(IndexConfig.createDefault(), 0, 1,
ImmutableSet.<String> of());
@@ -78,144 +69,13 @@ public class IndexedChangeQuery extends Predicate<ChangeData>
return IndexedChangeQuery.createOptions(opts.config(), 0, limit, opts.fields());
}
private final ChangeIndex index;
private QueryOptions opts;
private Predicate<ChangeData> pred;
private DataSource<ChangeData> source;
public IndexedChangeQuery(ChangeIndex index, Predicate<ChangeData> pred,
QueryOptions opts) throws QueryParseException {
this.index = index;
this.opts = convertOptions(opts);
this.pred = pred;
this.source = index.getSource(pred, this.opts);
}
@Override
public int getChildCount() {
return 1;
}
@Override
public Predicate<ChangeData> getChild(int i) {
if (i == 0) {
return pred;
}
throw new ArrayIndexOutOfBoundsException(i);
}
@Override
public List<Predicate<ChangeData>> getChildren() {
return ImmutableList.of(pred);
}
@Override
public QueryOptions getOptions() {
return opts;
}
@Override
public int getCardinality() {
return source != null ? source.getCardinality() : opts.limit();
super(index, pred, convertOptions(opts));
}
@Override
public boolean hasChange() {
return index.getSchema().hasField(ChangeField.CHANGE);
}
@Override
public ResultSet<ChangeData> read() throws OrmException {
final DataSource<ChangeData> currSource = source;
final ResultSet<ChangeData> rs = currSource.read();
return new ResultSet<ChangeData>() {
@Override
public Iterator<ChangeData> iterator() {
return Iterables.transform(
rs,
new Function<ChangeData, ChangeData>() {
@Override
public
ChangeData apply(ChangeData input) {
input.cacheFromSource(currSource);
return input;
}
}).iterator();
}
@Override
public List<ChangeData> toList() {
List<ChangeData> r = rs.toList();
for (ChangeData cd : r) {
cd.cacheFromSource(currSource);
}
return r;
}
@Override
public void close() {
rs.close();
}
};
}
@Override
public ResultSet<ChangeData> restart(int start) throws OrmException {
opts = opts.withStart(start);
try {
source = index.getSource(pred, opts);
} catch (QueryParseException e) {
// Don't need to show this exception to the user; the only thing that
// changed about pred was its start, and any other QPEs that might happen
// should have already thrown from the constructor.
throw new OrmException(e);
}
// Don't convert start to a limit, since the caller of this method (see
// AndSource) has calculated the actual number to skip.
return read();
}
@Override
public Predicate<ChangeData> copy(
Collection<? extends Predicate<ChangeData>> children) {
return this;
}
@Override
public boolean match(ChangeData cd) throws OrmException {
return (source != null && cd.isFromSource(source)) || pred.match(cd);
}
@Override
public int getCost() {
// Index queries are assumed to be cheaper than any other type of query, so
// so try to make sure they get picked. Note that pred's cost may be higher
// because it doesn't know whether it's being used in an index query or not.
return 1;
}
@Override
public int hashCode() {
return pred.hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null || getClass() != other.getClass()) {
return false;
}
IndexedChangeQuery o = (IndexedChangeQuery) other;
return pred.equals(o.pred)
&& opts.equals(o.opts);
}
@Override
public String toString() {
return MoreObjects.toStringHelper("index")
.add("p", pred)
.add("opts", opts)
.toString();
}
}

View File

@@ -12,31 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.change;
package com.google.gerrit.server.query;
import static com.google.gerrit.server.query.change.ChangeQueryBuilder.FIELD_LIMIT;
import com.google.gerrit.server.query.IntPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryBuilder;
import com.google.gerrit.server.query.QueryParseException;
public class LimitPredicate extends IntPredicate<ChangeData> {
public class LimitPredicate<T> extends IntPredicate<T> {
@SuppressWarnings("unchecked")
public static Integer getLimit(Predicate<ChangeData> p) {
IntPredicate<?> ip = QueryBuilder.find(p, IntPredicate.class, FIELD_LIMIT);
public static Integer getLimit(String fieldName, Predicate<?> p) {
IntPredicate<?> ip = QueryBuilder.find(p, IntPredicate.class, fieldName);
return ip != null ? ip.intValue() : null;
}
public LimitPredicate(int limit) throws QueryParseException {
super(ChangeQueryBuilder.FIELD_LIMIT, limit);
public LimitPredicate(String fieldName, int limit) throws QueryParseException {
super(fieldName, limit);
if (limit <= 0) {
throw new QueryParseException("limit must be positive: " + limit);
}
}
@Override
public boolean match(ChangeData object) {
public boolean match(T object) {
return true;
}

View File

@@ -12,14 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.change;
package com.google.gerrit.server.query;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
public interface Paginated {
public interface Paginated<T> {
QueryOptions getOptions();
ResultSet<ChangeData> restart(int start) throws OrmException;
ResultSet<T> restart(int start) throws OrmException;
}

View File

@@ -22,6 +22,7 @@ import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.gerrit.server.query.AndPredicate;
import com.google.gerrit.server.query.Paginated;
import com.google.gerrit.server.query.Predicate;
import com.google.gwtorm.server.ListResultSet;
import com.google.gwtorm.server.OrmException;
@@ -126,7 +127,8 @@ public class AndSource extends AndPredicate<ChangeData>
// least one of its results, we may not have filled the full
// limit the caller wants. Restart the source and continue.
//
Paginated p = (Paginated) source;
@SuppressWarnings("unchecked")
Paginated<ChangeData> p = (Paginated<ChangeData>) source;
while (skipped && r.size() < p.getOptions().limit() + start) {
skipped = false;
ResultSet<ChangeData> next = p.restart(nextStart);

View File

@@ -64,7 +64,6 @@ import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.SubmitRuleEvaluator;
import com.google.gerrit.server.query.DataSource;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.assistedinject.Assisted;
@@ -323,7 +322,6 @@ public class ChangeData {
private final MergeabilityCache mergeabilityCache;
private final StarredChangesUtil starredChangesUtil;
private final Change.Id legacyId;
private DataSource<ChangeData> returnedBySource;
private Project.NameKey project;
private Change change;
private ChangeNotes notes;
@@ -551,14 +549,6 @@ public class ChangeData {
return db;
}
public boolean isFromSource(DataSource<ChangeData> s) {
return s == returnedBySource;
}
public void cacheFromSource(DataSource<ChangeData> s) {
returnedBySource = s;
}
public void setCurrentFilePaths(List<String> filePaths) throws OrmException {
PatchSet ps = currentPatchSet();
if (ps != null) {

View File

@@ -57,12 +57,13 @@ import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.IndexRewriter;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.project.ListChildProjects;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.query.LimitPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryBuilder;
import com.google.gerrit.server.query.QueryParseException;
@@ -158,7 +159,7 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
public static class Arguments {
final Provider<ReviewDb> db;
final Provider<InternalChangeQuery> queryProvider;
final IndexRewriter rewriter;
final ChangeIndexRewriter rewriter;
final DynamicMap<ChangeOperatorFactory> opFactories;
final IdentifiedUser.GenericFactory userFactory;
final CapabilityControl.Factory capabilityControlFactory;
@@ -190,7 +191,7 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
@VisibleForTesting
public Arguments(Provider<ReviewDb> db,
Provider<InternalChangeQuery> queryProvider,
IndexRewriter rewriter,
ChangeIndexRewriter rewriter,
DynamicMap<ChangeOperatorFactory> opFactories,
IdentifiedUser.GenericFactory userFactory,
Provider<CurrentUser> self,
@@ -229,7 +230,7 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
private Arguments(
Provider<ReviewDb> db,
Provider<InternalChangeQuery> queryProvider,
IndexRewriter rewriter,
ChangeIndexRewriter rewriter,
DynamicMap<ChangeOperatorFactory> opFactories,
IdentifiedUser.GenericFactory userFactory,
Provider<CurrentUser> self,
@@ -843,7 +844,8 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
@Operator
public Predicate<ChangeData> limit(String limit) throws QueryParseException {
return new LimitPredicate(Integer.parseInt(limit));
return new LimitPredicate<>(ChangeQueryBuilder.FIELD_LIMIT,
Integer.parseInt(limit));
}
@Operator

View File

@@ -15,6 +15,7 @@
package com.google.gerrit.server.query.change;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.server.query.change.ChangeQueryBuilder.FIELD_LIMIT;
import static com.google.gerrit.server.query.change.ChangeStatusPredicate.open;
import com.google.common.base.Throwables;
@@ -32,10 +33,11 @@ import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.IndexRewriter;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.index.change.IndexedChangeQuery;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.query.LimitPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gwtorm.server.OrmException;
@@ -54,7 +56,7 @@ public class QueryProcessor {
private final ChangeControl.GenericFactory changeControlFactory;
private final ChangeNotes.Factory notesFactory;
private final ChangeIndexCollection indexes;
private final IndexRewriter rewriter;
private final ChangeIndexRewriter rewriter;
private final IndexConfig indexConfig;
private final Metrics metrics;
@@ -69,7 +71,7 @@ public class QueryProcessor {
ChangeControl.GenericFactory changeControlFactory,
ChangeNotes.Factory notesFactory,
ChangeIndexCollection indexes,
IndexRewriter rewriter,
ChangeIndexRewriter rewriter,
IndexConfig indexConfig,
Metrics metrics) {
this.db = db;
@@ -245,7 +247,7 @@ public class QueryProcessor {
if (limitFromCaller > 0) {
possibleLimits.add(limitFromCaller);
}
Integer limitFromPredicate = LimitPredicate.getLimit(p);
Integer limitFromPredicate = LimitPredicate.getLimit(FIELD_LIMIT, p);
if (limitFromPredicate != null) {
possibleLimits.add(limitFromPredicate);
}