opendev/gerrit
Code Issues Proposed changes

Merge changes from topic "project-index"

Browse Source 
* changes:
  Add inname, description and default field to project query builder
  ProjectIndexCollection: Remove unnecessary @Inject
  Convert project index to use ProjectData instead of ProjectState
  Add project index tests for both Lucene and ElasticSearch
  Add query methods to project API
  REST API support for project query
  Index project on creation and cache eviction
  Implementation of project index in Elastic Search
  Initial implementation of project index in Lucene
  Add ProjectQueryBuilder and define key predicate for project index
  Add schema and index definitions for project index
This commit is contained in:
Dave Borowitz
2017-10-05 18:18:55 +00:00
committed by Gerrit Code Review
parent f38ff5905a a2f3971161
commit bd3f218b8a
41 changed files with 2176 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Documentation/dev-plugins.txt
View File

@@ -407,6 +407,10 @@ Update of the account secondary index
+
Update of the group secondary index
* `com.google.gerrit.server.extensions.events.ProjectIndexedListener`:
+
Update of the project secondary index
* `com.google.gerrit.httpd.WebLoginListener`:
+
User login or logout interactively on the Web user interface.

59
Documentation/rest-api-projects.txt
View File

@@ -324,6 +324,65 @@ GET /projects/?type=PERMISSIONS HTTP/1.0
}
----
[[query-projects]]
=== Query Projects
--
'GET /projects/?query=<query>'
--
Queries projects visible to the caller. The
link:user-search-projects.html#_search_operators[query string] must be
provided by the `query` parameter. The `start` and `limit` parameters
can be used to skip/limit results.
As result a list of link:#project-info[ProjectInfo] entities is returned.
.Request
----
GET /projects/?query=name:test HTTP/1.0
----
.Response
----
HTTP/1.1 200 OK
Content-Disposition: attachment
Content-Type: application/json; charset=UTF-8
)]}'
{
"test": {
"id": "test",
"description": "\u003chtml\u003e is escaped"
}
}
----
[[project-query-limit]]
==== Project Limit
The `/projects/?query=<query>` URL also accepts a limit integer in the
`limit` parameter. This limits the results to `limit` projects.
Query the first 25 projects in project list.
----
GET /projects/?query=<query>&limit=25 HTTP/1.0
----
The `/projects/` URL also accepts a start integer in the `start`
parameter. The results will skip `start` groups from project list.
Query 25 projects starting from index 50.
----
GET /groups/?query=<query>&limit=25&start=50 HTTP/1.0
----
[[project-query-options]]
==== Project Options
Additional fields can be obtained by adding `o` parameters. Each option
requires more lookups and slows down the query response time to the
client so they are generally disabled by default. The supported fields
are described in the context of the link:#project-options[List Projects]
REST endpoint.
[[get-project]]
=== Get Project
--

36
Documentation/user-search-projects.txt Normal file
View File

@@ -0,0 +1,36 @@
= Gerrit Code Review - Searching Projects
[[search-operators]]
== Search Operators
Operators act as restrictions on the search. As more operators
are added to the same query string, they further restrict the
returned results.
[[name]]
name:'NAME'::
+
Matches projects that have the NAME 'NAME'.
== Magical Operators
[[is-visible]]
is:visible::
+
Magical internal flag to prove the current user has access to read
the projects and all the refs. This flag is always added to any query.
[[limit]]
limit:'CNT'::
+
Limit the returned results to no more than 'CNT' records. This is
automatically set to the page size configured in the current user's
preferences. Including it in a web query may lead to unpredictable
results with regards to pagination.
GERRIT
------
Part of link:index.html[Gerrit Code Review]
SEARCHBOX
---------

5
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java
View File

@@ -26,6 +26,7 @@ import com.google.gerrit.server.index.VersionManager;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
@@ -75,6 +76,10 @@ public class ElasticIndexModule extends AbstractModule {
new FactoryModuleBuilder()
.implement(GroupIndex.class, ElasticGroupIndex.class)
.build(GroupIndex.Factory.class));
install(
new FactoryModuleBuilder()
.implement(ProjectIndex.class, ElasticProjectIndex.class)
.build(ProjectIndex.Factory.class));
install(new IndexModule(threads));
if (singleVersions == null) {

215
gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java Normal file
View File

@@ -0,0 +1,215 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.project.ProjectField;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectData;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import io.searchbox.client.JestResult;
import io.searchbox.core.Bulk;
import io.searchbox.core.Bulk.Builder;
import io.searchbox.core.Search;
import io.searchbox.core.search.sort.Sort;
import io.searchbox.core.search.sort.Sort.Sorting;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, ProjectData>
implements ProjectIndex {
static class ProjectMapping {
MappingProperties projects;
ProjectMapping(Schema<ProjectData> schema) {
this.projects = ElasticMapping.createMapping(schema);
}
}
static final String PROJECTS = "projects";
static final String PROJECTS_PREFIX = PROJECTS + "_";
private static final Logger log = LoggerFactory.getLogger(ElasticProjectIndex.class);
private final ProjectMapping mapping;
private final Provider<ProjectCache> projectCache;
@Inject
ElasticProjectIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
JestClientBuilder clientBuilder,
@Assisted Schema<ProjectData> schema) {
super(cfg, sitePaths, schema, clientBuilder, PROJECTS_PREFIX);
this.projectCache = projectCache;
this.mapping = new ProjectMapping(schema);
}
@Override
public void replace(ProjectData projectState) throws IOException {
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(PROJECTS)
.addAction(insert(PROJECTS, projectState))
.refresh(true)
.build();
JestResult result = client.execute(bulk);
if (!result.isSucceeded()) {
throw new IOException(
String.format(
"Failed to replace project %s in index %s: %s",
projectState.getProject().getName(), indexName, result.getErrorMessage()));
}
}
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(p, opts);
}
@Override
protected Builder addActions(Builder builder, Project.NameKey nameKey) {
return builder.addAction(delete(PROJECTS, nameKey));
}
@Override
protected String getMappings() {
ImmutableMap<String, ProjectMapping> mappings = ImmutableMap.of("mappings", mapping);
return gson.toJson(mappings);
}
@Override
protected String getId(ProjectData projectState) {
return projectState.getProject().getName();
}
private class QuerySource implements DataSource<ProjectData> {
private final Search search;
private final Set<String> fields;
QuerySource(Predicate<ProjectData> p, QueryOptions opts) throws QueryParseException {
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
fields = IndexUtils.projectFields(opts);
SearchSourceBuilder searchSource =
new SearchSourceBuilder()
.query(qb)
.from(opts.start())
.size(opts.limit())
.fields(Lists.newArrayList(fields));
Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC);
sort.setIgnoreUnmapped();
search =
new Search.Builder(searchSource.toString())
.addType(PROJECTS)
.addIndex(indexName)
.addSort(ImmutableList.of(sort))
.build();
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<ProjectData> read() throws OrmException {
try {
List<ProjectData> results = Collections.emptyList();
JestResult result = client.execute(search);
if (result.isSucceeded()) {
JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
results = Lists.newArrayListWithCapacity(json.size());
for (int i = 0; i < json.size(); i++) {
results.add(toProjectData(json.get(i)));
}
}
} else {
log.error(result.getErrorMessage());
}
final List<ProjectData> r = Collections.unmodifiableList(results);
return new ResultSet<ProjectData>() {
@Override
public Iterator<ProjectData> iterator() {
return r.iterator();
}
@Override
public List<ProjectData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
}
}
@Override
public String toString() {
return search.toString();
}
private ProjectData toProjectData(JsonElement json) {
JsonElement source = json.getAsJsonObject().get("_source");
if (source == null) {
source = json.getAsJsonObject().get("fields");
}
Project.NameKey nameKey =
new Project.NameKey(
source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
return projectCache.get().get(nameKey).toProjectData();
}
}
}

65
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java Normal file
View File

@@ -0,0 +1,65 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.elasticsearch;
import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
import com.google.gerrit.server.query.project.AbstractQueryProjectsTest;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.Config;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
public class ElasticQueryProjectsTest extends AbstractQueryProjectsTest {
private static ElasticNodeInfo nodeInfo;
@BeforeClass
public static void startIndexService() throws InterruptedException, ExecutionException {
if (nodeInfo != null) {
// do not start Elasticsearch twice
return;
}
nodeInfo = ElasticTestUtils.startElasticsearchNode();
ElasticTestUtils.createAllIndexes(nodeInfo);
}
@AfterClass
public static void stopElasticsearchServer() {
if (nodeInfo != null) {
nodeInfo.node.close();
nodeInfo.elasticDir.delete();
nodeInfo = null;
}
}
@After
public void cleanupIndex() {
if (nodeInfo != null) {
ElasticTestUtils.deleteAllIndexes(nodeInfo);
ElasticTestUtils.createAllIndexes(nodeInfo);
}
}
@Override
protected Injector createInjector() {
Config elasticsearchConfig = new Config(config);
InMemoryModule.setDefaults(elasticsearchConfig);
ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration));
}
}

16
gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
View File

@@ -20,12 +20,14 @@ import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CHANGES_PREFIX;
import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CLOSED_CHANGES;
import static com.google.gerrit.elasticsearch.ElasticChangeIndex.OPEN_CHANGES;
import static com.google.gerrit.elasticsearch.ElasticGroupIndex.GROUPS_PREFIX;
import static com.google.gerrit.elasticsearch.ElasticProjectIndex.PROJECTS_PREFIX;
import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.gerrit.elasticsearch.ElasticAccountIndex.AccountMapping;
import com.google.gerrit.elasticsearch.ElasticChangeIndex.ChangeMapping;
import com.google.gerrit.elasticsearch.ElasticGroupIndex.GroupMapping;
import com.google.gerrit.elasticsearch.ElasticProjectIndex.ProjectMapping;
import com.google.gerrit.index.Schema;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.group.InternalGroup;
@@ -33,6 +35,8 @@ import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.gerrit.server.index.project.ProjectSchemaDefinitions;
import com.google.gerrit.server.project.ProjectData;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
@@ -157,6 +161,18 @@ final class ElasticTestUtils {
.addMapping(ElasticGroupIndex.GROUPS, gson.toJson(groupMapping))
.execute()
.actionGet();
Schema<ProjectData> projectSchema = ProjectSchemaDefinitions.INSTANCE.getLatest();
ProjectMapping projectMapping = new ProjectMapping(projectSchema);
nodeInfo
.node
.client()
.admin()
.indices()
.prepareCreate(String.format("%s%04d", PROJECTS_PREFIX, projectSchema.getVersion()))
.addMapping(ElasticProjectIndex.PROJECTS, gson.toJson(projectMapping))
.execute()
.actionGet();
}
private static String getHttpPort(Node node) throws InterruptedException, ExecutionException {

78
gerrit-extension-api/src/main/java/com/google/gerrit/extensions/api/projects/Projects.java
View File

@@ -58,6 +58,24 @@ public interface Projects {
ListRequest list();
/**
* Query projects.
*
* <p>Example code: {@code query().withQuery("name:project").get()}
*
* @return API for setting parameters and getting result.
*/
QueryRequest query();
/**
* Query projects.
*
* <p>Shortcut API for {@code query().withQuery(String)}.
*
* @see #query()
*/
QueryRequest query(String query);
abstract class ListRequest {
public enum FilterType {
CODE,
@@ -171,6 +189,56 @@ public interface Projects {
}
}
/**
* API for setting parameters and getting result. Used for {@code query()}.
*
* @see #query()
*/
abstract class QueryRequest {
private String query;
private int limit;
private int start;
/** Execute query and returns the matched projects as list. */
public abstract List<ProjectInfo> get() throws RestApiException;
/**
* Set query.
*
* @param query needs to be in human-readable form.
*/
public QueryRequest withQuery(String query) {
this.query = query;
return this;
}
/**
* Set limit for returned list of projects. Optional; server-default is used when not provided.
*/
public QueryRequest withLimit(int limit) {
this.limit = limit;
return this;
}
/** Set number of projects to skip. Optional; no projects are skipped when not provided. */
public QueryRequest withStart(int start) {
this.start = start;
return this;
}
public String getQuery() {
return query;
}
public int getLimit() {
return limit;
}
public int getStart() {
return start;
}
}
/**
* A default implementation which allows source compatibility when adding new methods to the
* interface.
@@ -195,5 +263,15 @@ public interface Projects {
public ListRequest list() {
throw new NotImplementedException();
}
@Override
public QueryRequest query() {
throw new NotImplementedException();
}
@Override
public QueryRequest query(String query) {
throw new NotImplementedException();
}
}
}

28
gerrit-extension-api/src/main/java/com/google/gerrit/extensions/events/ProjectIndexedListener.java Normal file
View File

@@ -0,0 +1,28 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.extensions.events;
import com.google.gerrit.extensions.annotations.ExtensionPoint;
/** Notified whenever a project is indexed */
@ExtensionPoint
public interface ProjectIndexedListener {
/**
* Invoked when a project is indexed
*
* @param name of the project
*/
void onProjectIndexed(String project);
}

6
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
View File

@@ -27,6 +27,7 @@ import com.google.gerrit.server.index.VersionManager;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
@@ -85,7 +86,10 @@ public class LuceneIndexModule extends AbstractModule {
new FactoryModuleBuilder()
.implement(GroupIndex.class, LuceneGroupIndex.class)
.build(GroupIndex.Factory.class));
install(
new FactoryModuleBuilder()
.implement(ProjectIndex.class, LuceneProjectIndex.class)
.build(ProjectIndex.Factory.class));
install(new IndexModule(threads));
if (singleVersions == null) {
install(new MultiVersionModule());

200
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java Normal file
View File

@@ -0,0 +1,200 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
import static com.google.gerrit.server.index.project.ProjectField.NAME;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectData;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.eclipse.jgit.lib.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class LuceneProjectIndex extends AbstractLuceneIndex<Project.NameKey, ProjectData>
implements ProjectIndex {
private static final Logger log = LoggerFactory.getLogger(LuceneProjectIndex.class);
private static final String PROJECTS = "projects";
private static final String NAME_SORT_FIELD = sortFieldName(NAME);
private static Term idTerm(ProjectData projectState) {
return idTerm(projectState.getProject().getNameKey());
}
private static Term idTerm(Project.NameKey nameKey) {
return QueryBuilder.stringTerm(NAME.getName(), nameKey.get());
}
private final GerritIndexWriterConfig indexWriterConfig;
private final QueryBuilder<ProjectData> queryBuilder;
private final Provider<ProjectCache> projectCache;
private static Directory dir(Schema<ProjectData> schema, Config cfg, SitePaths sitePaths)
throws IOException {
if (LuceneIndexModule.isInMemoryTest(cfg)) {
return new RAMDirectory();
}
Path indexDir = LuceneVersionManager.getDir(sitePaths, PROJECTS, schema);
return FSDirectory.open(indexDir);
}
@Inject
LuceneProjectIndex(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
@Assisted Schema<ProjectData> schema)
throws IOException {
super(
schema,
sitePaths,
dir(schema, cfg, sitePaths),
PROJECTS,
null,
new GerritIndexWriterConfig(cfg, PROJECTS),
new SearcherFactory());
this.projectCache = projectCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS);
queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer());
}
@Override
public void replace(ProjectData projectState) throws IOException {
try {
replace(idTerm(projectState), toDocument(projectState)).get();
} catch (ExecutionException | InterruptedException e) {
throw new IOException(e);
}
}
@Override
public void delete(Project.NameKey nameKey) throws IOException {
try {
delete(idTerm(nameKey)).get();
} catch (ExecutionException | InterruptedException e) {
throw new IOException(e);
}
}
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
return new QuerySource(
opts,
queryBuilder.toQuery(p),
new Sort(new SortField(NAME_SORT_FIELD, SortField.Type.STRING, false)));
}
private class QuerySource implements DataSource<ProjectData> {
private final QueryOptions opts;
private final Query query;
private final Sort sort;
private QuerySource(QueryOptions opts, Query query, Sort sort) {
this.opts = opts;
this.query = query;
this.sort = sort;
}
@Override
public int getCardinality() {
return 10;
}
@Override
public ResultSet<ProjectData> read() throws OrmException {
IndexSearcher searcher = null;
try {
searcher = acquire();
int realLimit = opts.start() + opts.limit();
TopFieldDocs docs = searcher.search(query, realLimit, sort);
List<ProjectData> result = new ArrayList<>(docs.scoreDocs.length);
for (int i = opts.start(); i < docs.scoreDocs.length; i++) {
ScoreDoc sd = docs.scoreDocs[i];
Document doc = searcher.doc(sd.doc, IndexUtils.projectFields(opts));
result.add(toProjectData(doc));
}
final List<ProjectData> r = Collections.unmodifiableList(result);
return new ResultSet<ProjectData>() {
@Override
public Iterator<ProjectData> iterator() {
return r.iterator();
}
@Override
public List<ProjectData> toList() {
return r;
}
@Override
public void close() {
// Do nothing.
}
};
} catch (IOException e) {
throw new OrmException(e);
} finally {
if (searcher != null) {
try {
release(searcher);
} catch (IOException e) {
log.warn("cannot release Lucene searcher", e);
}
}
}
}
}
private ProjectData toProjectData(Document doc) {
Project.NameKey nameKey = new Project.NameKey(doc.getField(NAME.getName()).stringValue());
return projectCache.get().get(nameKey).toProjectData();
}
}

37
gerrit-server/src/main/java/com/google/gerrit/server/api/projects/ProjectsImpl.java
View File

@@ -22,14 +22,18 @@ import com.google.gerrit.extensions.api.projects.Projects;
import com.google.gerrit.extensions.common.ProjectInfo;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.extensions.restapi.TopLevelResource;
import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.project.ListProjects;
import com.google.gerrit.server.project.ListProjects.FilterType;
import com.google.gerrit.server.project.ProjectsCollection;
import com.google.gerrit.server.project.QueryProjects;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.util.List;
import java.util.SortedMap;
@Singleton
@@ -37,15 +41,18 @@ class ProjectsImpl implements Projects {
private final ProjectsCollection projects;
private final ProjectApiImpl.Factory api;
private final Provider<ListProjects> listProvider;
private final Provider<QueryProjects> queryProvider;
@Inject
ProjectsImpl(
ProjectsCollection projects,
ProjectApiImpl.Factory api,
Provider<ListProjects> listProvider) {
Provider<ListProjects> listProvider,
Provider<QueryProjects> queryProvider) {
this.projects = projects;
this.api = api;
this.listProvider = listProvider;
this.queryProvider = queryProvider;
}
@Override
@@ -124,4 +131,32 @@ class ProjectsImpl implements Projects {
return lp.apply();
}
@Override
public QueryRequest query() {
return new QueryRequest() {
@Override
public List<ProjectInfo> get() throws RestApiException {
return ProjectsImpl.this.query(this);
}
};
}
@Override
public QueryRequest query(String query) {
return query().withQuery(query);
}
private List<ProjectInfo> query(QueryRequest r) throws RestApiException {
try {
QueryProjects myQueryProjects = queryProvider.get();
myQueryProjects.setQuery(r.getQuery());
myQueryProjects.setLimit(r.getLimit());
myQueryProjects.setStart(r.getStart());
return myQueryProjects.apply(TopLevelResource.INSTANCE);
} catch (OrmException e) {
throw new RestApiException("Cannot query projects", e);
}
}
}

6
gerrit-server/src/main/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
View File

@@ -2563,7 +2563,11 @@ class ReceiveCommits {
}
if (isConfig(cmd)) {
logDebug("Reloading project in cache");
projectCache.evict(project);
try {
projectCache.evict(project);
} catch (IOException e) {
log.warn("Cannot evict from project cache, name key: " + project.getName(), e);
}
ProjectState ps = projectCache.get(project.getNameKey());
try {
logDebug("Updating project description");

10
gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java
View File

@@ -23,6 +23,8 @@ import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.DummyChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.gerrit.server.project.ProjectData;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.AbstractModule;
@@ -48,6 +50,13 @@ public class DummyIndexModule extends AbstractModule {
}
}
private static class DummyProjectIndexFactory implements ProjectIndex.Factory {
@Override
public ProjectIndex create(Schema<ProjectData> schema) {
throw new UnsupportedOperationException();
}
}
@Override
protected void configure() {
install(new IndexModule(1));
@@ -56,5 +65,6 @@ public class DummyIndexModule extends AbstractModule {
bind(AccountIndex.Factory.class).toInstance(new DummyAccountIndexFactory());
bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory());
bind(GroupIndex.Factory.class).toInstance(new DummyGroupIndexFactory());
bind(ProjectIndex.Factory.class).toInstance(new DummyProjectIndexFactory());
}
}

28
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
View File

@@ -45,6 +45,12 @@ import com.google.gerrit.server.index.group.GroupIndexRewriter;
import com.google.gerrit.server.index.group.GroupIndexer;
import com.google.gerrit.server.index.group.GroupIndexerImpl;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.gerrit.server.index.project.ProjectIndexCollection;
import com.google.gerrit.server.index.project.ProjectIndexDefinition;
import com.google.gerrit.server.index.project.ProjectIndexRewriter;
import com.google.gerrit.server.index.project.ProjectIndexer;
import com.google.gerrit.server.index.project.ProjectIndexerImpl;
import com.google.gerrit.server.index.project.ProjectSchemaDefinitions;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provides;
@@ -70,7 +76,8 @@ public class IndexModule extends LifecycleModule {
ImmutableList.<SchemaDefinitions<?>>of(
AccountSchemaDefinitions.INSTANCE,
ChangeSchemaDefinitions.INSTANCE,
GroupSchemaDefinitions.INSTANCE);
GroupSchemaDefinitions.INSTANCE,
ProjectSchemaDefinitions.INSTANCE);
/** Type of secondary index. */
public static IndexType getIndexType(Injector injector) {
@@ -112,14 +119,22 @@ public class IndexModule extends LifecycleModule {
listener().to(GroupIndexCollection.class);
factory(GroupIndexerImpl.Factory.class);
bind(ProjectIndexRewriter.class);
bind(ProjectIndexCollection.class);
listener().to(ProjectIndexCollection.class);
factory(ProjectIndexerImpl.Factory.class);
DynamicSet.setOf(binder(), OnlineUpgradeListener.class);
}
@Provides
Collection<IndexDefinition<?, ?, ?>> getIndexDefinitions(
AccountIndexDefinition accounts, ChangeIndexDefinition changes, GroupIndexDefinition groups) {
AccountIndexDefinition accounts,
ChangeIndexDefinition changes,
GroupIndexDefinition groups,
ProjectIndexDefinition projects) {
Collection<IndexDefinition<?, ?, ?>> result =
ImmutableList.<IndexDefinition<?, ?, ?>>of(accounts, groups, changes);
ImmutableList.<IndexDefinition<?, ?, ?>>of(accounts, groups, changes, projects);
Set<String> expected =
FluentIterable.from(ALL_SCHEMA_DEFS).transform(SchemaDefinitions::getName).toSet();
Set<String> actual = FluentIterable.from(result).transform(IndexDefinition::getName).toSet();
@@ -154,6 +169,13 @@ public class IndexModule extends LifecycleModule {
return factory.create(indexes);
}
@Provides
@Singleton
ProjectIndexer getProjectIndexer(
ProjectIndexerImpl.Factory factory, ProjectIndexCollection indexes) {
return factory.create(indexes);
}
@Provides
@Singleton
@IndexExecutor(INTERACTIVE)

8
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java
View File

@@ -26,6 +26,7 @@ import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.account.AccountField;
import com.google.gerrit.server.index.group.GroupField;
import com.google.gerrit.server.index.project.ProjectField;
import com.google.gerrit.server.query.change.SingleGroupUser;
import java.io.IOException;
import java.util.Set;
@@ -94,6 +95,13 @@ public final class IndexUtils {
return user.toString();
}
public static Set<String> projectFields(QueryOptions opts) {
Set<String> fs = opts.fields();
return fs.contains(ProjectField.NAME.getName())
? fs
: Sets.union(fs, ImmutableSet.of(ProjectField.NAME.getName()));
}
private IndexUtils() {
// hide default constructor
}

120
gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java Normal file
View File

@@ -0,0 +1,120 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
import com.google.common.base.Stopwatch;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.index.SiteIndexer;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectData;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.TextProgressMonitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class AllProjectsIndexer extends SiteIndexer<Project.NameKey, ProjectData, ProjectIndex> {
private static final Logger log = LoggerFactory.getLogger(AllProjectsIndexer.class);
private final ListeningExecutorService executor;
private final ProjectCache projectCache;
@Inject
AllProjectsIndexer(
@IndexExecutor(BATCH) ListeningExecutorService executor, ProjectCache projectCache) {
this.executor = executor;
this.projectCache = projectCache;
}
@Override
public SiteIndexer.Result indexAll(final ProjectIndex index) {
ProgressMonitor progress = new TextProgressMonitor(new PrintWriter(progressOut));
progress.start(2);
Stopwatch sw = Stopwatch.createStarted();
List<Project.NameKey> names;
try {
names = collectProjects(progress);
} catch (OrmException e) {
log.error("Error collecting projects", e);
return new SiteIndexer.Result(sw, false, 0, 0);
}
return reindexProjects(index, names, progress);
}
private SiteIndexer.Result reindexProjects(
ProjectIndex index, List<Project.NameKey> names, ProgressMonitor progress) {
progress.beginTask("Reindexing projects", names.size());
List<ListenableFuture<?>> futures = new ArrayList<>(names.size());
AtomicBoolean ok = new AtomicBoolean(true);
AtomicInteger done = new AtomicInteger();
AtomicInteger failed = new AtomicInteger();
Stopwatch sw = Stopwatch.createStarted();
for (Project.NameKey name : names) {
String desc = "project " + name;
ListenableFuture<?> future =
executor.submit(
() -> {
try {
projectCache.evict(name);
index.replace(projectCache.get(name).toProjectData());
verboseWriter.println("Reindexed " + desc);
done.incrementAndGet();
} catch (Exception e) {
failed.incrementAndGet();
throw e;
}
return null;
});
addErrorListener(future, desc, progress, ok);
futures.add(future);
}
try {
Futures.successfulAsList(futures).get();
} catch (ExecutionException | InterruptedException e) {
log.error("Error waiting on project futures", e);
return new SiteIndexer.Result(sw, false, 0, 0);
}
progress.endTask();
return new SiteIndexer.Result(sw, ok.get(), done.get(), failed.get());
}
private List<Project.NameKey> collectProjects(ProgressMonitor progress) throws OrmException {
progress.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
List<Project.NameKey> names = new ArrayList<>();
for (Project.NameKey nameKey : projectCache.all()) {
names.add(nameKey);
}
progress.endTask();
return names;
}
}

34
gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java Normal file
View File

@@ -0,0 +1,34 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexedQuery;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.project.ProjectData;
public class IndexedProjectQuery extends IndexedQuery<Project.NameKey, ProjectData>
implements DataSource<ProjectData> {
public IndexedProjectQuery(
Index<Project.NameKey, ProjectData> index, Predicate<ProjectData> pred, QueryOptions opts)
throws QueryParseException {
super(index, pred, opts.convertForBackend());
}
}

44
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java Normal file
View File

@@ -0,0 +1,44 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import static com.google.gerrit.index.FieldDef.exact;
import static com.google.gerrit.index.FieldDef.fullText;
import static com.google.gerrit.index.FieldDef.prefix;
import com.google.common.collect.Iterables;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.SchemaUtil;
import com.google.gerrit.server.project.ProjectData;
/** Index schema for projects. */
public class ProjectField {
public static final FieldDef<ProjectData, String> NAME =
exact("name").stored().build(p -> p.getProject().getName());
public static final FieldDef<ProjectData, String> DESCRIPTION =
fullText("description").build(p -> p.getProject().getDescription());
public static final FieldDef<ProjectData, String> PARENT_NAME =
exact("parent_name").build(p -> p.getProject().getParentName());
public static final FieldDef<ProjectData, Iterable<String>> NAME_PART =
prefix("name_part").buildRepeatable(p -> SchemaUtil.getNameParts(p.getProject().getName()));
public static final FieldDef<ProjectData, Iterable<String>> ANCESTOR_NAME =
exact("ancestor_name")
.buildRepeatable(p -> Iterables.transform(p.getAncestors(), n -> n.get()));
}

33
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.project.ProjectData;
import com.google.gerrit.server.query.project.ProjectPredicates;
public interface ProjectIndex extends Index<Project.NameKey, ProjectData> {
public interface Factory
extends IndexDefinition.IndexFactory<Project.NameKey, ProjectData, ProjectIndex> {}
@Override
default Predicate<ProjectData> keyPredicate(Project.NameKey nameKey) {
return ProjectPredicates.name(nameKey);
}
}

29
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java Normal file
View File

@@ -0,0 +1,29 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import com.google.common.annotations.VisibleForTesting;
import com.google.gerrit.index.IndexCollection;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.project.ProjectData;
import com.google.inject.Singleton;
@Singleton
public class ProjectIndexCollection
extends IndexCollection<Project.NameKey, ProjectData, ProjectIndex> {
@VisibleForTesting
public ProjectIndexCollection() {}
}

33
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.project.ProjectData;
import com.google.inject.Inject;
public class ProjectIndexDefinition
extends IndexDefinition<Project.NameKey, ProjectData, ProjectIndex> {
@Inject
ProjectIndexDefinition(
ProjectIndexCollection indexCollection,
ProjectIndex.Factory indexFactory,
@Nullable AllProjectsIndexer allProjectsIndexer) {
super(ProjectSchemaDefinitions.INSTANCE, indexCollection, indexFactory, allProjectsIndexer);
}
}

43
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java Normal file
View File

@@ -0,0 +1,43 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.gerrit.index.IndexRewriter;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.server.project.ProjectData;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@Singleton
public class ProjectIndexRewriter implements IndexRewriter<ProjectData> {
private final ProjectIndexCollection indexes;
@Inject
ProjectIndexRewriter(ProjectIndexCollection indexes) {
this.indexes = indexes;
}
@Override
public Predicate<ProjectData> rewrite(Predicate<ProjectData> in, QueryOptions opts)
throws QueryParseException {
ProjectIndex index = indexes.getSearchIndex();
checkNotNull(index, "no active search index configured for projects");
return new IndexedProjectQuery(index, in, opts);
}
}

28
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexer.java Normal file
View File

@@ -0,0 +1,28 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import com.google.gerrit.reviewdb.client.Project;
import java.io.IOException;
public interface ProjectIndexer {
/**
* Synchronously index a project.
*
* @param nameKey name key of project to index.
*/
void index(Project.NameKey nameKey) throws IOException;
}

86
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java Normal file
View File

@@ -0,0 +1,86 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import com.google.common.collect.ImmutableSet;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.events.ProjectIndexedListener;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.index.Index;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectData;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
public class ProjectIndexerImpl implements ProjectIndexer {
public interface Factory {
ProjectIndexerImpl create(ProjectIndexCollection indexes);
ProjectIndexerImpl create(@Nullable ProjectIndex index);
}
private final ProjectCache projectCache;
private final DynamicSet<ProjectIndexedListener> indexedListener;
private final ProjectIndexCollection indexes;
private final ProjectIndex index;
@AssistedInject
ProjectIndexerImpl(
ProjectCache projectCache,
DynamicSet<ProjectIndexedListener> indexedListener,
@Assisted ProjectIndexCollection indexes) {
this.projectCache = projectCache;
this.indexedListener = indexedListener;
this.indexes = indexes;
this.index = null;
}
@AssistedInject
ProjectIndexerImpl(
ProjectCache projectCache,
DynamicSet<ProjectIndexedListener> indexedListener,
@Assisted ProjectIndex index) {
this.projectCache = projectCache;
this.indexedListener = indexedListener;
this.indexes = null;
this.index = index;
}
@Override
public void index(Project.NameKey nameKey) throws IOException {
for (Index<?, ProjectData> i : getWriteIndexes()) {
i.replace(projectCache.get(nameKey).toProjectData());
}
fireProjectIndexedEvent(nameKey.get());
}
private void fireProjectIndexedEvent(String name) {
for (ProjectIndexedListener listener : indexedListener) {
listener.onProjectIndexed(name);
}
}
private Collection<ProjectIndex> getWriteIndexes() {
if (indexes != null) {
return indexes.getWriteIndexes();
}
return index != null ? Collections.singleton(index) : ImmutableSet.of();
}
}

38
gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java Normal file
View File

@@ -0,0 +1,38 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index.project;
import static com.google.gerrit.index.SchemaUtil.schema;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.SchemaDefinitions;
import com.google.gerrit.server.project.ProjectData;
public class ProjectSchemaDefinitions extends SchemaDefinitions<ProjectData> {
static final Schema<ProjectData> V1 =
schema(
ProjectField.NAME,
ProjectField.DESCRIPTION,
ProjectField.PARENT_NAME,
ProjectField.NAME_PART,
ProjectField.ANCESTOR_NAME);
public static final ProjectSchemaDefinitions INSTANCE = new ProjectSchemaDefinitions();
private ProjectSchemaDefinitions() {
super("projects", ProjectData.class);
}
}

3
gerrit-server/src/main/java/com/google/gerrit/server/project/PerRequestProjectControlCache.java
View File

@@ -18,6 +18,7 @@ import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.CurrentUser;
import com.google.inject.Inject;
import com.google.inject.servlet.RequestScoped;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -48,7 +49,7 @@ public class PerRequestProjectControlCache {
return ctl;
}
public void evict(Project project) {
public void evict(Project project) throws IOException {
projectCache.evict(project);
controls.remove(project.getNameKey());
}

22
gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCache.java
View File

@@ -45,17 +45,27 @@ public interface ProjectCache {
*/
ProjectState checkedGet(Project.NameKey projectName) throws IOException;
/** Invalidate the cached information about the given project. */
void evict(Project p);
/**
* Invalidate the cached information about the given project, and triggers reindexing for it
*
* @param p project that is being evicted
* @throws IOException thrown if the reindexing fails
*/
void evict(Project p) throws IOException;
/** Invalidate the cached information about the given project. */
void evict(Project.NameKey p);
/**
* Invalidate the cached information about the given project, and triggers reindexing for it
*
* @param p the NameKey of the project that is being evicted
* @throws IOException thrown if the reindexing fails
*/
void evict(Project.NameKey p) throws IOException;
/**
* Remove information about the given project from the cache. It will no longer be returned from
* {@link #all()}.
*/
void remove(Project p);
void remove(Project p) throws IOException;
/** @return sorted iteration of projects. */
Iterable<Project.NameKey> all();
@@ -75,5 +85,5 @@ public interface ProjectCache {
Iterable<Project.NameKey> byName(String prefix);
/** Notify the cache that a new project was constructed. */
void onCreateProject(Project.NameKey newProjectName);
void onCreateProject(Project.NameKey newProjectName) throws IOException;
}

22
gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheImpl.java
View File

@@ -28,8 +28,10 @@ import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.ProjectConfig;
import com.google.gerrit.server.index.project.ProjectIndexer;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.internal.UniqueAnnotations;
@@ -82,6 +84,7 @@ public class ProjectCacheImpl implements ProjectCache {
private final LoadingCache<ListKey, SortedSet<Project.NameKey>> list;
private final Lock listLock;
private final ProjectCacheClock clock;
private final Provider<ProjectIndexer> indexer;
@Inject
ProjectCacheImpl(
@@ -89,13 +92,15 @@ public class ProjectCacheImpl implements ProjectCache {
final AllUsersName allUsersName,
@Named(CACHE_NAME) LoadingCache<String, ProjectState> byName,
@Named(CACHE_LIST) LoadingCache<ListKey, SortedSet<Project.NameKey>> list,
ProjectCacheClock clock) {
ProjectCacheClock clock,
Provider<ProjectIndexer> indexer) {
this.allProjectsName = allProjectsName;
this.allUsersName = allUsersName;
this.byName = byName;
this.list = list;
this.listLock = new ReentrantLock(true /* fair */);
this.clock = clock;
this.indexer = indexer;
}
@Override
@@ -151,22 +156,20 @@ public class ProjectCacheImpl implements ProjectCache {
}
@Override
public void evict(Project p) {
if (p != null) {
byName.invalidate(p.getNameKey().get());
}
public void evict(Project p) throws IOException {
evict(p.getNameKey());
}
/** Invalidate the cached information about the given project. */
@Override
public void evict(Project.NameKey p) {
public void evict(Project.NameKey p) throws IOException {
if (p != null) {
byName.invalidate(p.get());
}
indexer.get().index(p);
}
@Override
public void remove(Project p) {
public void remove(Project p) throws IOException {
listLock.lock();
try {
SortedSet<Project.NameKey> n = Sets.newTreeSet(list.get(ListKey.ALL));
@@ -181,7 +184,7 @@ public class ProjectCacheImpl implements ProjectCache {
}
@Override
public void onCreateProject(Project.NameKey newProjectName) {
public void onCreateProject(Project.NameKey newProjectName) throws IOException {
listLock.lock();
try {
SortedSet<Project.NameKey> n = Sets.newTreeSet(list.get(ListKey.ALL));
@@ -192,6 +195,7 @@ public class ProjectCacheImpl implements ProjectCache {
} finally {
listLock.unlock();
}
indexer.get().index(newProjectName);
}
@Override

4
gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectControl.java
View File

@@ -390,6 +390,10 @@ public class ProjectControl {
}
}
public boolean canRead() {
return !isHidden() && allRefsAreVisible(Collections.emptySet());
}
ForProject asForProject() {
return new ForProjectImpl();
}

36
gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectData.java Normal file
View File

@@ -0,0 +1,36 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.project;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.reviewdb.client.Project;
public class ProjectData {
private final Project project;
private final ImmutableList<Project.NameKey> ancestors;
public ProjectData(Project project, Iterable<Project.NameKey> ancestors) {
this.project = project;
this.ancestors = ImmutableList.copyOf(ancestors);
}
public Project getProject() {
return project;
}
public ImmutableList<Project.NameKey> getAncestors() {
return ancestors;
}
}

4
gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectState.java
View File

@@ -553,6 +553,10 @@ public class ProjectState {
}
}
public ProjectData toProjectData() {
return new ProjectData(getProject(), parents().transform(s -> s.getProject().getNameKey()));
}
private String readFile(Path p) throws IOException {
return Files.exists(p) ? new String(Files.readAllBytes(p), UTF_8) : null;
}

21
gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectsCollection.java
View File

@@ -14,11 +14,14 @@
package com.google.gerrit.server.project;
import com.google.common.collect.ListMultimap;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.registration.DynamicMap;
import com.google.gerrit.extensions.restapi.AcceptsCreate;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.IdString;
import com.google.gerrit.extensions.restapi.NeedsParams;
import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.extensions.restapi.RestCollection;
import com.google.gerrit.extensions.restapi.RestView;
@@ -38,32 +41,48 @@ import org.eclipse.jgit.lib.Constants;
@Singleton
public class ProjectsCollection
implements RestCollection<TopLevelResource, ProjectResource>, AcceptsCreate<TopLevelResource> {
implements RestCollection<TopLevelResource, ProjectResource>,
AcceptsCreate<TopLevelResource>,
NeedsParams {
private final DynamicMap<RestView<ProjectResource>> views;
private final Provider<ListProjects> list;
private final Provider<QueryProjects> queryProjects;
private final ProjectControl.GenericFactory controlFactory;
private final PermissionBackend permissionBackend;
private final Provider<CurrentUser> user;
private final CreateProject.Factory createProjectFactory;
private boolean hasQuery;
@Inject
ProjectsCollection(
DynamicMap<RestView<ProjectResource>> views,
Provider<ListProjects> list,
Provider<QueryProjects> queryProjects,
ProjectControl.GenericFactory controlFactory,
PermissionBackend permissionBackend,
CreateProject.Factory factory,
Provider<CurrentUser> user) {
this.views = views;
this.list = list;
this.queryProjects = queryProjects;
this.controlFactory = controlFactory;
this.permissionBackend = permissionBackend;
this.user = user;
this.createProjectFactory = factory;
}
@Override
public void setParams(ListMultimap<String, String> params) throws BadRequestException {
// The --query option is defined in QueryProjects
this.hasQuery = params.containsKey("query");
}
@Override
public RestView<TopLevelResource> list() {
if (hasQuery) {
return queryProjects.get();
}
return list.get().setFormat(OutputFormat.JSON);
}

120
gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java Normal file
View File

@@ -0,0 +1,120 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.project;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.gerrit.extensions.common.ProjectInfo;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.MethodNotAllowedException;
import com.google.gerrit.extensions.restapi.RestReadView;
import com.google.gerrit.extensions.restapi.TopLevelResource;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.index.query.QueryResult;
import com.google.gerrit.server.index.project.ProjectIndex;
import com.google.gerrit.server.index.project.ProjectIndexCollection;
import com.google.gerrit.server.query.project.ProjectQueryBuilder;
import com.google.gerrit.server.query.project.ProjectQueryProcessor;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import java.util.ArrayList;
import java.util.List;
import org.kohsuke.args4j.Option;
public class QueryProjects implements RestReadView<TopLevelResource> {
private final ProjectIndexCollection indexes;
private final ProjectQueryBuilder queryBuilder;
private final ProjectQueryProcessor queryProcessor;
private final ProjectJson json;
private String query;
private int limit;
private int start;
@Option(
name = "--query",
aliases = {"-q"},
usage = "project query"
)
public void setQuery(String query) {
this.query = query;
}
@Option(
name = "--limit",
aliases = {"-n"},
metaVar = "CNT",
usage = "maximum number of projects to list"
)
public void setLimit(int limit) {
this.limit = limit;
}
@Option(
name = "--start",
aliases = {"-S"},
metaVar = "CNT",
usage = "number of projects to skip"
)
public void setStart(int start) {
this.start = start;
}
@Inject
protected QueryProjects(
ProjectIndexCollection indexes,
ProjectQueryBuilder queryBuilder,
ProjectQueryProcessor queryProcessor,
ProjectJson json) {
this.indexes = indexes;
this.queryBuilder = queryBuilder;
this.queryProcessor = queryProcessor;
this.json = json;
}
@Override
public List<ProjectInfo> apply(TopLevelResource resource)
throws BadRequestException, MethodNotAllowedException, OrmException {
if (Strings.isNullOrEmpty(query)) {
throw new BadRequestException("missing query field");
}
ProjectIndex searchIndex = indexes.getSearchIndex();
if (searchIndex == null) {
throw new MethodNotAllowedException("no project index");
}
if (start != 0) {
queryProcessor.setStart(start);
}
if (limit != 0) {
queryProcessor.setUserProvidedLimit(limit);
}
try {
QueryResult<ProjectData> result = queryProcessor.query(queryBuilder.parse(query));
List<ProjectData> pds = result.entities();
ArrayList<ProjectInfo> projectInfos = Lists.newArrayListWithCapacity(pds.size());
for (ProjectData pd : pds) {
projectInfos.add(json.format(pd.getProject()));
}
return projectInfos;
} catch (QueryParseException e) {
throw new BadRequestException(e.getMessage());
}
}
}

48
gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java Normal file
View File

@@ -0,0 +1,48 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import com.google.gerrit.index.query.IsVisibleToPredicate;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.ProjectPermission;
import com.google.gerrit.server.project.ProjectData;
import com.google.gerrit.server.query.account.AccountQueryBuilder;
import com.google.gwtorm.server.OrmException;
public class ProjectIsVisibleToPredicate extends IsVisibleToPredicate<ProjectData> {
protected final PermissionBackend permissionBackend;
protected final CurrentUser user;
public ProjectIsVisibleToPredicate(PermissionBackend permissionBackend, CurrentUser user) {
super(AccountQueryBuilder.FIELD_VISIBLETO, IndexUtils.describe(user));
this.permissionBackend = permissionBackend;
this.user = user;
}
@Override
public boolean match(ProjectData pd) throws OrmException {
return permissionBackend
.user(user)
.project(pd.getProject().getNameKey())
.testOrFalse(ProjectPermission.READ);
}
@Override
public int getCost() {
return 1;
}
}

45
gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.index.project.ProjectField;
import com.google.gerrit.server.project.ProjectData;
import java.util.Locale;
public class ProjectPredicates {
public static Predicate<ProjectData> name(Project.NameKey nameKey) {
return new ProjectPredicate(ProjectField.NAME, nameKey.get());
}
public static Predicate<ProjectData> inname(String name) {
return new ProjectPredicate(ProjectField.NAME_PART, name.toLowerCase(Locale.US));
}
public static Predicate<ProjectData> description(String description) {
return new ProjectPredicate(ProjectField.DESCRIPTION, description);
}
static class ProjectPredicate extends IndexPredicate<ProjectData> {
ProjectPredicate(FieldDef<ProjectData, ?> def, String value) {
super(def, value);
}
}
private ProjectPredicates() {}
}

83
gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java Normal file
View File

@@ -0,0 +1,83 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.primitives.Ints;
import com.google.gerrit.index.query.LimitPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryBuilder;
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.project.ProjectData;
import com.google.inject.Inject;
import java.util.List;
/** Parses a query string meant to be applied to project objects. */
public class ProjectQueryBuilder extends QueryBuilder<ProjectData> {
public static final String FIELD_LIMIT = "limit";
private static final QueryBuilder.Definition<ProjectData, ProjectQueryBuilder> mydef =
new QueryBuilder.Definition<>(ProjectQueryBuilder.class);
@Inject
ProjectQueryBuilder() {
super(mydef);
}
@Operator
public Predicate<ProjectData> name(String name) {
return ProjectPredicates.name(new Project.NameKey(name));
}
@Operator
public Predicate<ProjectData> inname(String namePart) {
if (namePart.isEmpty()) {
return name(namePart);
}
return ProjectPredicates.inname(namePart);
}
@Operator
public Predicate<ProjectData> description(String description) throws QueryParseException {
if (Strings.isNullOrEmpty(description)) {
throw error("description operator requires a value");
}
return ProjectPredicates.description(description);
}
@Override
protected Predicate<ProjectData> defaultField(String query) throws QueryParseException {
// Adapt the capacity of this list when adding more default predicates.
List<Predicate<ProjectData>> preds = Lists.newArrayListWithCapacity(3);
preds.add(name(query));
preds.add(inname(query));
if (!Strings.isNullOrEmpty(query)) {
preds.add(description(query));
}
return Predicate.or(preds);
}
@Operator
public Predicate<ProjectData> limit(String query) throws QueryParseException {
Integer limit = Ints.tryParse(query);
if (limit == null) {
throw error("Invalid limit: " + query);
}
return new LimitPredicate<>(FIELD_LIMIT, limit);
}
}

79
gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java Normal file
View File

@@ -0,0 +1,79 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.server.query.project.ProjectQueryBuilder.FIELD_LIMIT;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.query.AndSource;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryProcessor;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.account.AccountLimits;
import com.google.gerrit.server.index.project.ProjectIndexCollection;
import com.google.gerrit.server.index.project.ProjectIndexRewriter;
import com.google.gerrit.server.index.project.ProjectSchemaDefinitions;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.project.ProjectData;
import com.google.inject.Inject;
import com.google.inject.Provider;
/**
* Query processor for the project index.
*
* <p>Instances are one-time-use. Other singleton classes should inject a Provider rather than
* holding on to a single instance.
*/
public class ProjectQueryProcessor extends QueryProcessor<ProjectData> {
private final PermissionBackend permissionBackend;
private final Provider<CurrentUser> userProvider;
static {
// It is assumed that basic rewrites do not touch visibleto predicates.
checkState(
!ProjectIsVisibleToPredicate.class.isAssignableFrom(IndexPredicate.class),
"ProjectQueryProcessor assumes visibleto is not used by the index rewriter.");
}
@Inject
protected ProjectQueryProcessor(
Provider<CurrentUser> userProvider,
AccountLimits.Factory limitsFactory,
MetricMaker metricMaker,
IndexConfig indexConfig,
ProjectIndexCollection indexes,
ProjectIndexRewriter rewriter,
PermissionBackend permissionBackend) {
super(
metricMaker,
ProjectSchemaDefinitions.INSTANCE,
indexConfig,
indexes,
rewriter,
FIELD_LIMIT,
() -> limitsFactory.create(userProvider.get()).getQueryLimit());
this.permissionBackend = permissionBackend;
this.userProvider = userProvider;
}
@Override
protected Predicate<ProjectData> enforceVisibility(Predicate<ProjectData> pred) {
return new AndSource<>(
pred, new ProjectIsVisibleToPredicate(permissionBackend, userProvider.get()), start);
}
}

372
gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java Normal file
View File

@@ -0,0 +1,372 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import static com.google.common.truth.Truth.assertThat;
import static java.util.stream.Collectors.toList;
import com.google.common.base.CharMatcher;
import com.google.gerrit.extensions.api.GerritApi;
import com.google.gerrit.extensions.api.projects.ProjectInput;
import com.google.gerrit.extensions.api.projects.Projects.QueryRequest;
import com.google.gerrit.extensions.common.AccountInfo;
import com.google.gerrit.extensions.common.ProjectInfo;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.lifecycle.LifecycleManager;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.AnonymousUser;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.account.AccountManager;
import com.google.gerrit.server.account.Accounts;
import com.google.gerrit.server.account.AccountsUpdate;
import com.google.gerrit.server.account.AuthRequest;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.query.account.InternalAccountQuery;
import com.google.gerrit.server.schema.SchemaCreator;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
import com.google.gerrit.server.util.RequestContext;
import com.google.gerrit.server.util.ThreadLocalRequestContext;
import com.google.gerrit.testutil.ConfigSuite;
import com.google.gerrit.testutil.GerritServerTests;
import com.google.gerrit.testutil.InMemoryDatabase;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
import com.google.inject.util.Providers;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import org.eclipse.jgit.lib.Config;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@Ignore
public abstract class AbstractQueryProjectsTest extends GerritServerTests {
@ConfigSuite.Default
public static Config defaultConfig() {
Config cfg = new Config();
cfg.setInt("index", null, "maxPages", 10);
return cfg;
}
@Inject protected Accounts accounts;
@Inject protected AccountsUpdate.Server accountsUpdate;
@Inject protected AccountCache accountCache;
@Inject protected AccountManager accountManager;
@Inject protected GerritApi gApi;
@Inject protected IdentifiedUser.GenericFactory userFactory;
@Inject private Provider<AnonymousUser> anonymousUser;
@Inject protected InMemoryDatabase schemaFactory;
@Inject protected SchemaCreator schemaCreator;
@Inject protected ThreadLocalRequestContext requestContext;
@Inject protected OneOffRequestContext oneOffRequestContext;
@Inject protected InternalAccountQuery internalAccountQuery;
@Inject protected AllProjectsName allProjects;
protected LifecycleManager lifecycle;
protected Injector injector;
protected ReviewDb db;
protected AccountInfo currentUserInfo;
protected CurrentUser user;
protected abstract Injector createInjector();
@Before
public void setUpInjector() throws Exception {
lifecycle = new LifecycleManager();
injector = createInjector();
lifecycle.add(injector);
injector.injectMembers(this);
lifecycle.start();
setUpDatabase();
}
protected void setUpDatabase() throws Exception {
db = schemaFactory.open();
schemaCreator.create(db);
Account.Id userId = createAccount("user", "User", "user@example.com", true);
user = userFactory.create(userId);
requestContext.setContext(newRequestContext(userId));
currentUserInfo = gApi.accounts().id(userId.get()).get();
}
protected RequestContext newRequestContext(Account.Id requestUserId) {
final CurrentUser requestUser = userFactory.create(requestUserId);
return new RequestContext() {
@Override
public CurrentUser getUser() {
return requestUser;
}
@Override
public Provider<ReviewDb> getReviewDbProvider() {
return Providers.of(db);
}
};
}
protected void setAnonymous() {
requestContext.setContext(
new RequestContext() {
@Override
public CurrentUser getUser() {
return anonymousUser.get();
}
@Override
public Provider<ReviewDb> getReviewDbProvider() {
return Providers.of(db);
}
});
}
@After
public void tearDownInjector() {
if (lifecycle != null) {
lifecycle.stop();
}
requestContext.setContext(null);
if (db != null) {
db.close();
}
InMemoryDatabase.drop(schemaFactory);
}
@Test
public void byName() throws Exception {
assertQuery("name:project");
assertQuery("name:non-existing");
ProjectInfo project = createProject(name("project"));
assertQuery("name:" + project.name, project);
// only exact match
ProjectInfo projectWithHyphen = createProject(name("project-with-hyphen"));
createProject(name("project-no-match-with-hyphen"));
assertQuery("name:" + projectWithHyphen.name, projectWithHyphen);
}
@Test
public void byInname() throws Exception {
String namePart = getSanitizedMethodName();
namePart = CharMatcher.is('_').removeFrom(namePart);
ProjectInfo project1 = createProject(name("project-" + namePart));
ProjectInfo project2 = createProject(name("project-" + namePart + "-2"));
ProjectInfo project3 = createProject(name("project-" + namePart + "3"));
assertQuery("inname:" + namePart, project1, project2, project3);
assertQuery("inname:" + namePart.toUpperCase(Locale.US), project1, project2, project3);
assertQuery("inname:" + namePart.toLowerCase(Locale.US), project1, project2, project3);
}
@Test
public void byDescription() throws Exception {
ProjectInfo project1 =
createProjectWithDescription(name("project1"), "This is a test project.");
ProjectInfo project2 = createProjectWithDescription(name("project2"), "ANOTHER TEST PROJECT.");
createProjectWithDescription(name("project3"), "Maintainers of project foo.");
assertQuery("description:test", project1, project2);
assertQuery("description:non-existing");
exception.expect(BadRequestException.class);
exception.expectMessage("description operator requires a value");
assertQuery("description:\"\"");
}
@Test
public void byDefaultField() throws Exception {
ProjectInfo project1 = createProject(name("foo-project"));
ProjectInfo project2 = createProject(name("project2"));
ProjectInfo project3 =
createProjectWithDescription(
name("project3"),
"decription that contains foo and the UUID of project2: " + project2.id);
assertQuery("non-existing");
assertQuery("foo", project1, project3);
assertQuery(project2.id, project2, project3);
}
@Test
public void withLimit() throws Exception {
ProjectInfo project1 = createProject(name("project1"));
ProjectInfo project2 = createProject(name("project2"));
ProjectInfo project3 = createProject(name("project3"));
String query =
"name:" + project1.name + " OR name:" + project2.name + " OR name:" + project3.name;
List<ProjectInfo> result = assertQuery(query, project1, project2, project3);
result = assertQuery(newQuery(query).withLimit(2), result.subList(0, 2));
}
@Test
public void withStart() throws Exception {
ProjectInfo project1 = createProject(name("project1"));
ProjectInfo project2 = createProject(name("project2"));
ProjectInfo project3 = createProject(name("project3"));
String query =
"name:" + project1.name + " OR name:" + project2.name + " OR name:" + project3.name;
List<ProjectInfo> result = assertQuery(query, project1, project2, project3);
assertQuery(newQuery(query).withStart(1), result.subList(1, 3));
}
@Test
public void asAnonymous() throws Exception {
ProjectInfo project = createProject(name("project"));
setAnonymous();
assertQuery("name:" + project.name);
}
private Account.Id createAccount(String username, String fullName, String email, boolean active)
throws Exception {
try (ManualRequestContext ctx = oneOffRequestContext.open()) {
Account.Id id = accountManager.authenticate(AuthRequest.forUser(username)).getAccountId();
if (email != null) {
accountManager.link(id, AuthRequest.forEmail(email));
}
accountsUpdate
.create()
.update(
id,
a -> {
a.setFullName(fullName);
a.setPreferredEmail(email);
a.setActive(active);
});
return id;
}
}
protected ProjectInfo createProject(String name) throws Exception {
ProjectInput in = new ProjectInput();
in.name = name;
return gApi.projects().create(in).get();
}
protected ProjectInfo createProjectWithDescription(String name, String description)
throws Exception {
ProjectInput in = new ProjectInput();
in.name = name;
in.description = description;
return gApi.projects().create(in).get();
}
protected ProjectInfo getProject(Project.NameKey nameKey) throws Exception {
return gApi.projects().name(nameKey.get()).get();
}
protected List<ProjectInfo> assertQuery(Object query, ProjectInfo... projects) throws Exception {
return assertQuery(newQuery(query), projects);
}
protected List<ProjectInfo> assertQuery(QueryRequest query, ProjectInfo... projects)
throws Exception {
return assertQuery(query, Arrays.asList(projects));
}
protected List<ProjectInfo> assertQuery(QueryRequest query, List<ProjectInfo> projects)
throws Exception {
List<ProjectInfo> result = query.get();
Iterable<String> names = names(result);
assertThat(names)
.named(format(query, result, projects))
.containsExactlyElementsIn(names(projects));
return result;
}
protected QueryRequest newQuery(Object query) {
return gApi.projects().query(query.toString());
}
protected String format(
QueryRequest query, List<ProjectInfo> actualProjects, List<ProjectInfo> expectedProjects) {
StringBuilder b = new StringBuilder();
b.append("query '").append(query.getQuery()).append("' with expected projects ");
b.append(format(expectedProjects));
b.append(" and result ");
b.append(format(actualProjects));
return b.toString();
}
protected String format(Iterable<ProjectInfo> projects) {
StringBuilder b = new StringBuilder();
b.append("[");
Iterator<ProjectInfo> it = projects.iterator();
while (it.hasNext()) {
ProjectInfo p = it.next();
b.append("{")
.append(p.id)
.append(", ")
.append("name=")
.append(p.name)
.append(", ")
.append("parent=")
.append(p.parent)
.append(", ")
.append("description=")
.append(p.description)
.append("}");
if (it.hasNext()) {
b.append(", ");
}
}
b.append("]");
return b.toString();
}
protected static Iterable<String> names(ProjectInfo... projects) {
return names(Arrays.asList(projects));
}
protected static Iterable<String> names(List<ProjectInfo> projects) {
return projects.stream().map(p -> p.name).collect(toList());
}
protected String name(String name) {
if (name == null) {
return null;
}
return name + "_" + getSanitizedMethodName();
}
}

44
gerrit-server/src/test/java/com/google/gerrit/server/query/project/LuceneQueryProjectsTest.java Normal file
View File

@@ -0,0 +1,44 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import com.google.gerrit.server.index.project.ProjectSchemaDefinitions;
import com.google.gerrit.testutil.ConfigSuite;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.gerrit.testutil.IndexVersions;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.util.List;
import java.util.Map;
import org.eclipse.jgit.lib.Config;
public class LuceneQueryProjectsTest extends AbstractQueryProjectsTest {
@ConfigSuite.Configs
public static Map<String, Config> againstPreviousIndexVersion() {
// the current schema version is already tested by the inherited default config suite
List<Integer> schemaVersions =
IndexVersions.getWithoutLatest(
com.google.gerrit.server.index.project.ProjectSchemaDefinitions.INSTANCE);
return IndexVersions.asConfigMap(
ProjectSchemaDefinitions.INSTANCE, schemaVersions, "againstIndexVersion", defaultConfig());
}
@Override
protected Injector createInjector() {
Config luceneConfig = new Config(config);
InMemoryModule.setDefaults(luceneConfig);
return Guice.createInjector(new InMemoryModule(luceneConfig, notesMigration));
}
}

8
gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/AdminSetParent.java
View File

@@ -174,7 +174,13 @@ final class AdminSetParent extends SshCommand {
err.append("error: ").append(msg).append("\n");
}
projectCache.evict(nameKey);
try {
projectCache.evict(nameKey);
} catch (IOException e) {
final String msg = "Cannot reindex project: " + name;
log.error(msg, e);
err.append("error: ").append(msg).append("\n");
}
}
if (err.length() > 0) {