opendev/gerrit
Code Issues Proposed changes

Move QueryProcessor and InternalQuery to gerrit-index

Browse Source 
These classes are necessary to build a useful index without requiring a
dependency on //gerrit-server:server. It does require a dependency on
//gerrit-server:metrics, which is a much smaller target (and arguably
doesn't belong in gerrit-server at all, but we're not fixing that now).

Change-Id: I2f0a88c315153e5d2b150ccdbddf97e3cba23a4a
This commit is contained in:
Dave Borowitz
2017-08-09 12:57:53 -04:00
parent cab1222270
commit 7b126ee4f8
9 changed files with 9 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
gerrit-server/src/main/java/com/google/gerrit/server/account/AccountLimits.java
View File

@@ -17,11 +17,11 @@ package com.google.gerrit.server.account;
import com.google.gerrit.common.data.GlobalCapability;
import com.google.gerrit.common.data.PermissionRange;
import com.google.gerrit.common.data.PermissionRule;
import com.google.gerrit.index.query.QueryProcessor;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.git.QueueProvider;
import com.google.gerrit.server.group.SystemGroupBackend;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.query.QueryProcessor;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.util.ArrayList;

105
gerrit-server/src/main/java/com/google/gerrit/server/query/InternalQuery.java
View File

@@ -1,105 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexCollection;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.index.query.QueryResult;
import com.google.gwtorm.server.OrmException;
import java.util.List;
import java.util.Set;
/**
* Execute a single query over a secondary index, for use by Gerrit internals.
*
* <p>By default, visibility of returned entities is not enforced (unlike in {@link
* QueryProcessor}). The methods in this class are not typically used by user-facing paths, but
* rather by internal callers that need to process all matching results.
*
* <p>Instances are one-time-use. Other singleton classes should inject a Provider rather than
* holding on to a single instance.
*/
public class InternalQuery<T> {
private final QueryProcessor<T> queryProcessor;
private final IndexCollection<?, T, ? extends Index<?, T>> indexes;
protected final IndexConfig indexConfig;
protected InternalQuery(
QueryProcessor<T> queryProcessor,
IndexCollection<?, T, ? extends Index<?, T>> indexes,
IndexConfig indexConfig) {
this.queryProcessor = queryProcessor.enforceVisibility(false);
this.indexes = indexes;
this.indexConfig = indexConfig;
}
public InternalQuery<T> setLimit(int n) {
queryProcessor.setUserProvidedLimit(n);
return this;
}
public InternalQuery<T> enforceVisibility(boolean enforce) {
queryProcessor.enforceVisibility(enforce);
return this;
}
public InternalQuery<T> setRequestedFields(Set<String> fields) {
queryProcessor.setRequestedFields(fields);
return this;
}
public InternalQuery<T> noFields() {
queryProcessor.setRequestedFields(ImmutableSet.<String>of());
return this;
}
public List<T> query(Predicate<T> p) throws OrmException {
try {
return queryProcessor.query(p).entities();
} catch (QueryParseException e) {
throw new OrmException(e);
}
}
/**
* Run multiple queries in parallel.
*
* <p>If a limit was specified using {@link #setLimit(int)}, that limit is applied to each query
* independently.
*
* @param queries list of queries.
* @return results of the queries, one list of results per input query, in the same order as the
* input.
*/
public List<List<T>> query(List<Predicate<T>> queries) throws OrmException {
try {
return Lists.transform(queryProcessor.query(queries), QueryResult::entities);
} catch (QueryParseException e) {
throw new OrmException(e);
}
}
protected Schema<T> schema() {
Index<?, T> index = indexes != null ? indexes.getSearchIndex() : null;
return index != null ? index.getSchema() : null;
}
}

340
gerrit-server/src/main/java/com/google/gerrit/server/query/QueryProcessor.java
View File

@@ -1,340 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Ordering;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexCollection;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.IndexRewriter;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.SchemaDefinitions;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.LimitPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.index.query.QueryResult;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.Field;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.metrics.Timer1;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.OrmRuntimeException;
import com.google.gwtorm.server.ResultSet;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.IntSupplier;
import java.util.stream.IntStream;
/**
* Lower-level implementation for executing a single query over a secondary index.
*
* <p>Instances are one-time-use. Other singleton classes should inject a Provider rather than
* holding on to a single instance.
*/
public abstract class QueryProcessor<T> {
protected static class Metrics {
final Timer1<String> executionTime;
Metrics(MetricMaker metricMaker) {
Field<String> index = Field.ofString("index", "index name");
executionTime =
metricMaker.newTimer(
"query/query_latency",
new Description("Successful query latency, accumulated over the life of the process")
.setCumulative()
.setUnit(Description.Units.MILLISECONDS),
index);
}
}
private final Metrics metrics;
private final SchemaDefinitions<T> schemaDef;
private final IndexConfig indexConfig;
private final IndexCollection<?, T, ? extends Index<?, T>> indexes;
private final IndexRewriter<T> rewriter;
private final String limitField;
private final IntSupplier permittedLimit;
// This class is not generally thread-safe, but programmer error may result in it being shared
// across threads. At least ensure the bit for checking if it's been used is threadsafe.
private final AtomicBoolean used;
protected int start;
private boolean enforceVisibility = true;
private int userProvidedLimit;
private Set<String> requestedFields;
protected QueryProcessor(
MetricMaker metricMaker,
SchemaDefinitions<T> schemaDef,
IndexConfig indexConfig,
IndexCollection<?, T, ? extends Index<?, T>> indexes,
IndexRewriter<T> rewriter,
String limitField,
IntSupplier permittedLimit) {
this.metrics = new Metrics(metricMaker);
this.schemaDef = schemaDef;
this.indexConfig = indexConfig;
this.indexes = indexes;
this.rewriter = rewriter;
this.limitField = limitField;
this.permittedLimit = permittedLimit;
this.used = new AtomicBoolean(false);
}
public QueryProcessor<T> setStart(int n) {
start = n;
return this;
}
/**
* Specify whether to enforce visibility by filtering out results that are not visible to the
* user.
*
* <p>Enforcing visibility may have performance consequences, as the index system may need to
* post-filter a large number of results to fill even a modest limit.
*
* <p>If visibility is enforced, the user's {@code queryLimit} global capability is also used to
* bound the total number of results. If this capability is non-positive, this results in the
* entire query processor being {@link #isDisabled() disabled}.
*
* @param enforce whether to enforce visibility.
* @return this.
*/
public QueryProcessor<T> enforceVisibility(boolean enforce) {
enforceVisibility = enforce;
return this;
}
/**
* Set an end-user-provided limit on the number of results returned.
*
* <p>Since this limit is provided by an end user, it may exceed the limit that they are
* authorized to use. This is allowed; the processor will take multiple possible limits into
* account and choose the one that makes the most sense.
*
* @param n limit; zero or negative means no limit.
* @return this.
*/
public QueryProcessor<T> setUserProvidedLimit(int n) {
userProvidedLimit = n;
return this;
}
public QueryProcessor<T> setRequestedFields(Set<String> fields) {
requestedFields = fields;
return this;
}
/**
* Query for entities that match a structured query.
*
* @see #query(List)
* @param query the query.
* @return results of the query.
*/
public QueryResult<T> query(Predicate<T> query) throws OrmException, QueryParseException {
return query(ImmutableList.of(query)).get(0);
}
/**
* Perform multiple queries in parallel.
*
* <p>If querying is disabled, short-circuits the index and returns empty results. Callers that
* wish to distinguish this case from a query returning no results from the index may call {@link
* #isDisabled()} themselves.
*
* @param queries list of queries.
* @return results of the queries, one QueryResult per input query, in the same order as the
* input.
*/
public List<QueryResult<T>> query(List<Predicate<T>> queries)
throws OrmException, QueryParseException {
try {
return query(null, queries);
} catch (OrmRuntimeException e) {
throw new OrmException(e.getMessage(), e);
} catch (OrmException e) {
if (e.getCause() != null) {
Throwables.throwIfInstanceOf(e.getCause(), QueryParseException.class);
}
throw e;
}
}
private List<QueryResult<T>> query(
@Nullable List<String> queryStrings, List<Predicate<T>> queries)
throws OrmException, QueryParseException {
long startNanos = System.nanoTime();
checkState(!used.getAndSet(true), "%s has already been used", getClass().getSimpleName());
int cnt = queries.size();
if (queryStrings != null) {
int qs = queryStrings.size();
checkArgument(qs == cnt, "got %s query strings but %s predicates", qs, cnt);
}
if (cnt == 0) {
return ImmutableList.of();
}
if (isDisabled()) {
return disabledResults(queryStrings, queries);
}
// Parse and rewrite all queries.
List<Integer> limits = new ArrayList<>(cnt);
List<Predicate<T>> predicates = new ArrayList<>(cnt);
List<DataSource<T>> sources = new ArrayList<>(cnt);
for (Predicate<T> q : queries) {
int limit = getEffectiveLimit(q);
limits.add(limit);
if (limit == getBackendSupportedLimit()) {
limit--;
}
int page = (start / limit) + 1;
if (page > indexConfig.maxPages()) {
throw new QueryParseException(
"Cannot go beyond page " + indexConfig.maxPages() + " of results");
}
// Always bump limit by 1, even if this results in exceeding the permitted
// max for this user. The only way to see if there are more entities is to
// ask for one more result from the query.
QueryOptions opts = createOptions(indexConfig, start, limit + 1, getRequestedFields());
Predicate<T> pred = rewriter.rewrite(q, opts);
if (enforceVisibility) {
pred = enforceVisibility(pred);
}
predicates.add(pred);
@SuppressWarnings("unchecked")
DataSource<T> s = (DataSource<T>) pred;
sources.add(s);
}
// Run each query asynchronously, if supported.
List<ResultSet<T>> matches = new ArrayList<>(cnt);
for (DataSource<T> s : sources) {
matches.add(s.read());
}
List<QueryResult<T>> out = new ArrayList<>(cnt);
for (int i = 0; i < cnt; i++) {
out.add(
QueryResult.create(
queryStrings != null ? queryStrings.get(i) : null,
predicates.get(i),
limits.get(i),
matches.get(i).toList()));
}
// Only measure successful queries that actually touched the index.
metrics.executionTime.record(
schemaDef.getName(), System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
return out;
}
private static <T> ImmutableList<QueryResult<T>> disabledResults(
List<String> queryStrings, List<Predicate<T>> queries) {
return IntStream.range(0, queries.size())
.mapToObj(
i ->
QueryResult.create(
queryStrings != null ? queryStrings.get(i) : null,
queries.get(i),
0,
ImmutableList.of()))
.collect(toImmutableList());
}
protected QueryOptions createOptions(
IndexConfig indexConfig, int start, int limit, Set<String> requestedFields) {
return QueryOptions.create(indexConfig, start, limit, requestedFields);
}
/**
* Invoked after the query was rewritten. Subclasses must overwrite this method to filter out
* results that are not visible to the calling user.
*
* @param pred the query
* @return the modified query
*/
protected abstract Predicate<T> enforceVisibility(Predicate<T> pred);
private Set<String> getRequestedFields() {
if (requestedFields != null) {
return requestedFields;
}
Index<?, T> index = indexes.getSearchIndex();
return index != null ? index.getSchema().getStoredFields().keySet() : ImmutableSet.<String>of();
}
/**
* Check whether querying should be disabled.
*
* <p>Currently, the only condition that can disable the whole query processor is if both {@link
* #enforceVisibility(boolean) visibility is enforced} and the user has a non-positive maximum
* value for the {@code queryLimit} capability.
*
* <p>If querying is disabled, all calls to {@link #query(Predicate)} and {@link #query(List)}
* will return empty results. This method can be used if callers wish to distinguish this case
* from a query returning no results from the index.
*
* @return true if querying should be disabled.
*/
public boolean isDisabled() {
return enforceVisibility && getPermittedLimit() <= 0;
}
private int getPermittedLimit() {
return enforceVisibility ? permittedLimit.getAsInt() : Integer.MAX_VALUE;
}
private int getBackendSupportedLimit() {
return indexConfig.maxLimit();
}
private int getEffectiveLimit(Predicate<T> p) {
List<Integer> possibleLimits = new ArrayList<>(4);
possibleLimits.add(getBackendSupportedLimit());
possibleLimits.add(getPermittedLimit());
if (userProvidedLimit > 0) {
possibleLimits.add(userProvidedLimit);
}
if (limitField != null) {
Integer limitFromPredicate = LimitPredicate.getLimit(limitField, p);
if (limitFromPredicate != null) {
possibleLimits.add(limitFromPredicate);
}
}
int result = Ordering.natural().min(possibleLimits);
// Should have short-circuited from #query or thrown some other exception before getting here.
checkState(result > 0, "effective limit should be positive");
return result;
}
}

2
gerrit-server/src/main/java/com/google/gerrit/server/query/account/AccountQueryProcessor.java
View File

@@ -21,6 +21,7 @@ import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.query.AndSource;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryProcessor;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.account.AccountControl;
@@ -29,7 +30,6 @@ import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.account.AccountIndexCollection;
import com.google.gerrit.server.index.account.AccountIndexRewriter;
import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
import com.google.gerrit.server.query.QueryProcessor;
import com.google.inject.Inject;
import com.google.inject.Provider;

2
gerrit-server/src/main/java/com/google/gerrit/server/query/account/InternalAccountQuery.java
View File

@@ -21,11 +21,11 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.query.InternalQuery;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.account.externalids.ExternalId;
import com.google.gerrit.server.index.account.AccountIndexCollection;
import com.google.gerrit.server.query.InternalQuery;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import java.util.Arrays;

2
gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeQueryProcessor.java
View File

@@ -23,6 +23,7 @@ import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryProcessor;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CurrentUser;
@@ -34,7 +35,6 @@ import com.google.gerrit.server.index.change.IndexedChangeQuery;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.query.QueryProcessor;
import com.google.inject.Inject;
import com.google.inject.Provider;
import java.util.ArrayList;

2
gerrit-server/src/main/java/com/google/gerrit/server/query/change/InternalChangeQuery.java
View File

@@ -25,6 +25,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.query.InternalQuery;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.reviewdb.client.Branch;
import com.google.gerrit.reviewdb.client.Change;
@@ -33,7 +34,6 @@ import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.query.InternalQuery;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import java.io.IOException;

2
gerrit-server/src/main/java/com/google/gerrit/server/query/group/GroupQueryProcessor.java
View File

@@ -21,6 +21,7 @@ import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.query.AndSource;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryProcessor;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.server.CurrentUser;
@@ -29,7 +30,6 @@ import com.google.gerrit.server.account.GroupControl;
import com.google.gerrit.server.index.group.GroupIndexCollection;
import com.google.gerrit.server.index.group.GroupIndexRewriter;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.gerrit.server.query.QueryProcessor;
import com.google.inject.Inject;
import com.google.inject.Provider;