From e6cd6b2c54cbf774f85121f5a531781f810b2b06 Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Thu, 1 Jun 2017 14:02:36 -0700 Subject: [PATCH 01/11] Add schema and index definitions for project index Change-Id: I5326c770c02d8bb3d7e283124207caa9e8d28581 --- .../index/project/AllProjectsIndexer.java | 120 ++++++++++++++++++ .../server/index/project/ProjectField.java | 45 +++++++ .../server/index/project/ProjectIndex.java | 26 ++++ .../index/project/ProjectIndexCollection.java | 31 +++++ .../index/project/ProjectIndexDefinition.java | 33 +++++ .../project/ProjectSchemaDefinitions.java | 38 ++++++ 6 files changed, 293 insertions(+) create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java new file mode 100644 index 0000000000..932396559d --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java @@ -0,0 +1,120 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH; + +import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.gerrit.index.SiteIndexer; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.index.IndexExecutor; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.project.ProjectState; +import com.google.gwtorm.server.OrmException; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class AllProjectsIndexer extends SiteIndexer { + + private static final Logger log = LoggerFactory.getLogger(AllProjectsIndexer.class); + + private final ListeningExecutorService executor; + private final ProjectCache projectCache; + + @Inject + AllProjectsIndexer( + @IndexExecutor(BATCH) ListeningExecutorService executor, ProjectCache projectCache) { + this.executor = executor; + this.projectCache = projectCache; + } + + @Override + public SiteIndexer.Result indexAll(final ProjectIndex index) { + ProgressMonitor progress = new TextProgressMonitor(new PrintWriter(progressOut)); + progress.start(2); + Stopwatch sw = Stopwatch.createStarted(); + List names; + try { + names = collectProjects(progress); + } catch (OrmException e) { + log.error("Error collecting projects", e); + return new SiteIndexer.Result(sw, false, 0, 0); + } + return reindexProjects(index, names, progress); + } + + private SiteIndexer.Result reindexProjects( + ProjectIndex index, List names, ProgressMonitor progress) { + progress.beginTask("Reindexing projects", names.size()); + List> futures = new ArrayList<>(names.size()); + AtomicBoolean ok = new AtomicBoolean(true); + AtomicInteger done = new AtomicInteger(); + AtomicInteger failed = new AtomicInteger(); + Stopwatch sw = Stopwatch.createStarted(); + for (Project.NameKey name : names) { + String desc = "project " + name; + ListenableFuture future = + executor.submit( + () -> { + try { + projectCache.evict(name); + index.replace(projectCache.get(name)); + verboseWriter.println("Reindexed " + desc); + done.incrementAndGet(); + } catch (Exception e) { + failed.incrementAndGet(); + throw e; + } + return null; + }); + addErrorListener(future, desc, progress, ok); + futures.add(future); + } + + try { + Futures.successfulAsList(futures).get(); + } catch (ExecutionException | InterruptedException e) { + log.error("Error waiting on project futures", e); + return new SiteIndexer.Result(sw, false, 0, 0); + } + + progress.endTask(); + return new SiteIndexer.Result(sw, ok.get(), done.get(), failed.get()); + } + + private List collectProjects(ProgressMonitor progress) throws OrmException { + progress.beginTask("Collecting projects", ProgressMonitor.UNKNOWN); + List names = new ArrayList<>(); + for (Project.NameKey nameKey : projectCache.all()) { + names.add(nameKey); + } + progress.endTask(); + return names; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java new file mode 100644 index 0000000000..86156e1be1 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java @@ -0,0 +1,45 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import static com.google.gerrit.index.FieldDef.exact; +import static com.google.gerrit.index.FieldDef.fullText; +import static com.google.gerrit.index.FieldDef.prefix; + +import com.google.common.collect.Iterables; +import com.google.gerrit.index.FieldDef; +import com.google.gerrit.index.SchemaUtil; +import com.google.gerrit.server.project.ProjectState; + +/** Index schema for projects. */ +public class ProjectField { + + public static final FieldDef NAME = + exact("name").build(p -> p.getProject().getName()); + + public static final FieldDef DESCRIPTION = + fullText("description").build(p -> p.getProject().getDescription()); + + public static final FieldDef PARENT_NAME = + exact("parent_name").build(p -> p.getProject().getParentName()); + + public static final FieldDef> NAME_PART = + prefix("name_part").buildRepeatable(p -> SchemaUtil.getNameParts(p.getProject().getName())); + + public static final FieldDef> ANCESTOR_NAME = + exact("ancestor_name") + .buildRepeatable( + p -> Iterables.transform(p.parents(), parent -> parent.getProject().getName())); +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java new file mode 100644 index 0000000000..8827085383 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java @@ -0,0 +1,26 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.project.ProjectState; + +public interface ProjectIndex extends Index { + + public interface Factory + extends IndexDefinition.IndexFactory {} +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java new file mode 100644 index 0000000000..89f2619823 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java @@ -0,0 +1,31 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.common.annotations.VisibleForTesting; +import com.google.gerrit.index.IndexCollection; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.project.ProjectState; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class ProjectIndexCollection + extends IndexCollection { + + @Inject + @VisibleForTesting + public ProjectIndexCollection() {} +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java new file mode 100644 index 0000000000..8cdd28a1d5 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java @@ -0,0 +1,33 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.gerrit.common.Nullable; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.project.ProjectState; +import com.google.inject.Inject; + +public class ProjectIndexDefinition + extends IndexDefinition { + + @Inject + ProjectIndexDefinition( + ProjectIndexCollection indexCollection, + ProjectIndex.Factory indexFactory, + @Nullable AllProjectsIndexer allProjectsIndexer) { + super(ProjectSchemaDefinitions.INSTANCE, indexCollection, indexFactory, allProjectsIndexer); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java new file mode 100644 index 0000000000..4ff6db197a --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java @@ -0,0 +1,38 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import static com.google.gerrit.index.SchemaUtil.schema; + +import com.google.gerrit.index.Schema; +import com.google.gerrit.index.SchemaDefinitions; +import com.google.gerrit.server.project.ProjectState; + +public class ProjectSchemaDefinitions extends SchemaDefinitions { + + static final Schema V1 = + schema( + ProjectField.NAME, + ProjectField.DESCRIPTION, + ProjectField.PARENT_NAME, + ProjectField.NAME_PART, + ProjectField.ANCESTOR_NAME); + + public static final ProjectSchemaDefinitions INSTANCE = new ProjectSchemaDefinitions(); + + private ProjectSchemaDefinitions() { + super("projects", ProjectState.class); + } +} From 42064e711fa3986326121c6033c05e1635ebefdf Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Wed, 12 Jul 2017 15:48:59 -0700 Subject: [PATCH 02/11] Add ProjectQueryBuilder and define key predicate for project index Change-Id: I5ab054c3dade3dc965db040be537d21cdfa728a9 --- .../server/index/project/ProjectIndex.java | 7 ++++ .../query/project/ProjectPredicates.java | 36 ++++++++++++++++++ .../query/project/ProjectQueryBuilder.java | 37 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java index 8827085383..80f4fa6184 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java @@ -16,11 +16,18 @@ package com.google.gerrit.server.index.project; import com.google.gerrit.index.Index; import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.index.query.Predicate; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.query.project.ProjectPredicates; public interface ProjectIndex extends Index { public interface Factory extends IndexDefinition.IndexFactory {} + + @Override + default Predicate keyPredicate(Project.NameKey nameKey) { + return ProjectPredicates.name(nameKey); + } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java new file mode 100644 index 0000000000..45e65d03bc --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java @@ -0,0 +1,36 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.project; + +import com.google.gerrit.index.FieldDef; +import com.google.gerrit.index.query.IndexPredicate; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.index.project.ProjectField; +import com.google.gerrit.server.project.ProjectState; + +public class ProjectPredicates { + public static Predicate name(Project.NameKey nameKey) { + return new ProjectPredicate(ProjectField.NAME, nameKey.get()); + } + + static class ProjectPredicate extends IndexPredicate { + ProjectPredicate(FieldDef def, String value) { + super(def, value); + } + } + + private ProjectPredicates() {} +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java new file mode 100644 index 0000000000..f3efdc183d --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java @@ -0,0 +1,37 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.project; + +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryBuilder; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.project.ProjectState; +import com.google.inject.Inject; + +/** Parses a query string meant to be applied to project objects. */ +public class ProjectQueryBuilder extends QueryBuilder { + private static final QueryBuilder.Definition mydef = + new QueryBuilder.Definition<>(ProjectQueryBuilder.class); + + @Inject + ProjectQueryBuilder() { + super(mydef); + } + + @Operator + public Predicate name(String name) { + return ProjectPredicates.name(new Project.NameKey(name)); + } +} From c74d8ec8a789f32612b0174720d5861032658767 Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Fri, 14 Jul 2017 15:39:38 -0700 Subject: [PATCH 03/11] Initial implementation of project index in Lucene Change-Id: I688c80aa3ccaa4ff87eb7929d8370eb22fad4f5d --- .../gerrit/lucene/LuceneIndexModule.java | 6 +- .../gerrit/lucene/LuceneProjectIndex.java | 200 ++++++++++++++++++ .../gerrit/server/index/DummyIndexModule.java | 10 + .../gerrit/server/index/IndexUtils.java | 8 + 4 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java index d738540e25..d5d6360612 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -27,6 +27,7 @@ import com.google.gerrit.server.index.VersionManager; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.project.ProjectIndex; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; @@ -85,7 +86,10 @@ public class LuceneIndexModule extends AbstractModule { new FactoryModuleBuilder() .implement(GroupIndex.class, LuceneGroupIndex.class) .build(GroupIndex.Factory.class)); - + install( + new FactoryModuleBuilder() + .implement(ProjectIndex.class, LuceneProjectIndex.class) + .build(ProjectIndex.Factory.class)); install(new IndexModule(threads)); if (singleVersions == null) { install(new MultiVersionModule()); diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java new file mode 100644 index 0000000000..544649f054 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java @@ -0,0 +1,200 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.lucene; + +import static com.google.gerrit.server.index.project.ProjectField.NAME; + +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.Schema; +import com.google.gerrit.index.query.DataSource; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryParseException; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.project.ProjectIndex; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.project.ProjectState; +import com.google.gwtorm.server.OrmException; +import com.google.gwtorm.server.ResultSet; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.assistedinject.Assisted; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; +import org.eclipse.jgit.lib.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LuceneProjectIndex extends AbstractLuceneIndex + implements ProjectIndex { + private static final Logger log = LoggerFactory.getLogger(LuceneProjectIndex.class); + + private static final String PROJECTS = "projects"; + + private static final String NAME_SORT_FIELD = sortFieldName(NAME); + + private static Term idTerm(ProjectState projectState) { + return idTerm(projectState.getProject().getNameKey()); + } + + private static Term idTerm(Project.NameKey nameKey) { + return QueryBuilder.stringTerm(NAME.getName(), nameKey.get()); + } + + private final GerritIndexWriterConfig indexWriterConfig; + private final QueryBuilder queryBuilder; + private final Provider projectCache; + + private static Directory dir(Schema schema, Config cfg, SitePaths sitePaths) + throws IOException { + if (LuceneIndexModule.isInMemoryTest(cfg)) { + return new RAMDirectory(); + } + Path indexDir = LuceneVersionManager.getDir(sitePaths, PROJECTS, schema); + return FSDirectory.open(indexDir); + } + + @Inject + LuceneProjectIndex( + @GerritServerConfig Config cfg, + SitePaths sitePaths, + Provider projectCache, + @Assisted Schema schema) + throws IOException { + super( + schema, + sitePaths, + dir(schema, cfg, sitePaths), + PROJECTS, + null, + new GerritIndexWriterConfig(cfg, PROJECTS), + new SearcherFactory()); + this.projectCache = projectCache; + + indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS); + queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer()); + } + + @Override + public void replace(ProjectState projectState) throws IOException { + try { + replace(idTerm(projectState), toDocument(projectState)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new IOException(e); + } + } + + @Override + public void delete(Project.NameKey nameKey) throws IOException { + try { + delete(idTerm(nameKey)).get(); + } catch (ExecutionException | InterruptedException e) { + throw new IOException(e); + } + } + + @Override + public DataSource getSource(Predicate p, QueryOptions opts) + throws QueryParseException { + return new QuerySource( + opts, + queryBuilder.toQuery(p), + new Sort(new SortField(NAME_SORT_FIELD, SortField.Type.STRING, false))); + } + + private class QuerySource implements DataSource { + private final QueryOptions opts; + private final Query query; + private final Sort sort; + + private QuerySource(QueryOptions opts, Query query, Sort sort) { + this.opts = opts; + this.query = query; + this.sort = sort; + } + + @Override + public int getCardinality() { + return 10; + } + + @Override + public ResultSet read() throws OrmException { + IndexSearcher searcher = null; + try { + searcher = acquire(); + int realLimit = opts.start() + opts.limit(); + TopFieldDocs docs = searcher.search(query, realLimit, sort); + List result = new ArrayList<>(docs.scoreDocs.length); + for (int i = opts.start(); i < docs.scoreDocs.length; i++) { + ScoreDoc sd = docs.scoreDocs[i]; + Document doc = searcher.doc(sd.doc, IndexUtils.projectFields(opts)); + result.add(toProjectState(doc)); + } + final List r = Collections.unmodifiableList(result); + return new ResultSet() { + @Override + public Iterator iterator() { + return r.iterator(); + } + + @Override + public List toList() { + return r; + } + + @Override + public void close() { + // Do nothing. + } + }; + } catch (IOException e) { + throw new OrmException(e); + } finally { + if (searcher != null) { + try { + release(searcher); + } catch (IOException e) { + log.warn("cannot release Lucene searcher", e); + } + } + } + } + } + + private ProjectState toProjectState(Document doc) { + Project.NameKey nameKey = new Project.NameKey(doc.getField(NAME.getName()).stringValue()); + return projectCache.get().get(nameKey); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java index 481726b59b..19d2bb61f4 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java @@ -23,6 +23,8 @@ import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.change.DummyChangeIndex; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.project.ProjectIndex; +import com.google.gerrit.server.project.ProjectState; import com.google.gerrit.server.query.change.ChangeData; import com.google.inject.AbstractModule; @@ -48,6 +50,13 @@ public class DummyIndexModule extends AbstractModule { } } + private static class DummyProjectIndexFactory implements ProjectIndex.Factory { + @Override + public ProjectIndex create(Schema schema) { + throw new UnsupportedOperationException(); + } + } + @Override protected void configure() { install(new IndexModule(1)); @@ -56,5 +65,6 @@ public class DummyIndexModule extends AbstractModule { bind(AccountIndex.Factory.class).toInstance(new DummyAccountIndexFactory()); bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory()); bind(GroupIndex.Factory.class).toInstance(new DummyGroupIndexFactory()); + bind(ProjectIndex.Factory.class).toInstance(new DummyProjectIndexFactory()); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java index ea9900b7ac..b37ed61c54 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java @@ -26,6 +26,7 @@ import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.account.AccountField; import com.google.gerrit.server.index.group.GroupField; +import com.google.gerrit.server.index.project.ProjectField; import com.google.gerrit.server.query.change.SingleGroupUser; import java.io.IOException; import java.util.Set; @@ -94,6 +95,13 @@ public final class IndexUtils { return user.toString(); } + public static Set projectFields(QueryOptions opts) { + Set fs = opts.fields(); + return fs.contains(ProjectField.NAME.getName()) + ? fs + : Sets.union(fs, ImmutableSet.of(ProjectField.NAME.getName())); + } + private IndexUtils() { // hide default constructor } From dea36d8efbdd5fee1d781f20e8ad6a246dda6aff Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Mon, 17 Jul 2017 11:49:33 -0700 Subject: [PATCH 04/11] Implementation of project index in Elastic Search Change-Id: I4596071b0d8ef62bf691d8b77b76fdf827ed1b69 --- .../elasticsearch/ElasticIndexModule.java | 5 + .../elasticsearch/ElasticProjectIndex.java | 215 ++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java index 7868443e0a..2d04e11ee9 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java @@ -26,6 +26,7 @@ import com.google.gerrit.server.index.VersionManager; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.project.ProjectIndex; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; @@ -75,6 +76,10 @@ public class ElasticIndexModule extends AbstractModule { new FactoryModuleBuilder() .implement(GroupIndex.class, ElasticGroupIndex.class) .build(GroupIndex.Factory.class)); + install( + new FactoryModuleBuilder() + .implement(ProjectIndex.class, ElasticProjectIndex.class) + .build(ProjectIndex.Factory.class)); install(new IndexModule(threads)); if (singleVersions == null) { diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java new file mode 100644 index 0000000000..c983a44872 --- /dev/null +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java @@ -0,0 +1,215 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.Schema; +import com.google.gerrit.index.query.DataSource; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryParseException; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.project.ProjectField; +import com.google.gerrit.server.index.project.ProjectIndex; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.project.ProjectState; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gwtorm.server.OrmException; +import com.google.gwtorm.server.ResultSet; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.assistedinject.Assisted; +import io.searchbox.client.JestResult; +import io.searchbox.core.Bulk; +import io.searchbox.core.Bulk.Builder; +import io.searchbox.core.Search; +import io.searchbox.core.search.sort.Sort; +import io.searchbox.core.search.sort.Sort.Sorting; +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import org.eclipse.jgit.lib.Config; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ElasticProjectIndex extends AbstractElasticIndex + implements ProjectIndex { + static class ProjectMapping { + MappingProperties projects; + + ProjectMapping(Schema schema) { + this.projects = ElasticMapping.createMapping(schema); + } + } + + static final String PROJECTS = "projects"; + static final String PROJECTS_PREFIX = PROJECTS + "_"; + + private static final Logger log = LoggerFactory.getLogger(ElasticProjectIndex.class); + + private final ProjectMapping mapping; + private final Provider projectCache; + + @Inject + ElasticProjectIndex( + @GerritServerConfig Config cfg, + SitePaths sitePaths, + Provider projectCache, + JestClientBuilder clientBuilder, + @Assisted Schema schema) { + super(cfg, sitePaths, schema, clientBuilder, PROJECTS_PREFIX); + this.projectCache = projectCache; + this.mapping = new ProjectMapping(schema); + } + + @Override + public void replace(ProjectState projectState) throws IOException { + Bulk bulk = + new Bulk.Builder() + .defaultIndex(indexName) + .defaultType(PROJECTS) + .addAction(insert(PROJECTS, projectState)) + .refresh(true) + .build(); + JestResult result = client.execute(bulk); + if (!result.isSucceeded()) { + throw new IOException( + String.format( + "Failed to replace project %s in index %s: %s", + projectState.getProject().getName(), indexName, result.getErrorMessage())); + } + } + + @Override + public DataSource getSource(Predicate p, QueryOptions opts) + throws QueryParseException { + return new QuerySource(p, opts); + } + + @Override + protected Builder addActions(Builder builder, Project.NameKey nameKey) { + return builder.addAction(delete(PROJECTS, nameKey)); + } + + @Override + protected String getMappings() { + ImmutableMap mappings = ImmutableMap.of("mappings", mapping); + return gson.toJson(mappings); + } + + @Override + protected String getId(ProjectState projectState) { + return projectState.getProject().getName(); + } + + private class QuerySource implements DataSource { + private final Search search; + private final Set fields; + + QuerySource(Predicate p, QueryOptions opts) throws QueryParseException { + QueryBuilder qb = queryBuilder.toQueryBuilder(p); + fields = IndexUtils.projectFields(opts); + SearchSourceBuilder searchSource = + new SearchSourceBuilder() + .query(qb) + .from(opts.start()) + .size(opts.limit()) + .fields(Lists.newArrayList(fields)); + + Sort sort = new Sort(ProjectField.NAME.getName(), Sorting.ASC); + sort.setIgnoreUnmapped(); + + search = + new Search.Builder(searchSource.toString()) + .addType(PROJECTS) + .addIndex(indexName) + .addSort(ImmutableList.of(sort)) + .build(); + } + + @Override + public int getCardinality() { + return 10; + } + + @Override + public ResultSet read() throws OrmException { + try { + List results = Collections.emptyList(); + JestResult result = client.execute(search); + if (result.isSucceeded()) { + JsonObject obj = result.getJsonObject().getAsJsonObject("hits"); + if (obj.get("hits") != null) { + JsonArray json = obj.getAsJsonArray("hits"); + results = Lists.newArrayListWithCapacity(json.size()); + for (int i = 0; i < json.size(); i++) { + results.add(toProjectState(json.get(i))); + } + } + } else { + log.error(result.getErrorMessage()); + } + final List r = Collections.unmodifiableList(results); + return new ResultSet() { + @Override + public Iterator iterator() { + return r.iterator(); + } + + @Override + public List toList() { + return r; + } + + @Override + public void close() { + // Do nothing. + } + }; + } catch (IOException e) { + throw new OrmException(e); + } + } + + @Override + public String toString() { + return search.toString(); + } + + private ProjectState toProjectState(JsonElement json) { + JsonElement source = json.getAsJsonObject().get("_source"); + if (source == null) { + source = json.getAsJsonObject().get("fields"); + } + + Project.NameKey nameKey = + new Project.NameKey( + source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString()); + return projectCache.get().get(nameKey); + } + } +} From 9716986b341533e9737effd6026f437aa2ab1f62 Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Thu, 13 Jul 2017 17:31:16 -0700 Subject: [PATCH 05/11] Index project on creation and cache eviction Change-Id: I5b737803dae56331845c46719dc30de99ad6ff27 --- Documentation/dev-plugins.txt | 4 + .../events/ProjectIndexedListener.java | 28 ++++++ .../server/git/receive/ReceiveCommits.java | 6 +- .../gerrit/server/index/IndexModule.java | 28 +++++- .../index/project/IndexedProjectQuery.java | 34 ++++++++ .../index/project/ProjectIndexRewriter.java | 43 ++++++++++ .../server/index/project/ProjectIndexer.java | 28 ++++++ .../index/project/ProjectIndexerImpl.java | 86 +++++++++++++++++++ .../PerRequestProjectControlCache.java | 3 +- .../gerrit/server/project/ProjectCache.java | 22 +++-- .../server/project/ProjectCacheImpl.java | 22 +++-- .../gerrit/server/project/ProjectControl.java | 4 + .../project/ProjectIsVisibleToPredicate.java | 48 +++++++++++ .../query/project/ProjectQueryBuilder.java | 14 +++ .../query/project/ProjectQueryProcessor.java | 79 +++++++++++++++++ .../gerrit/sshd/commands/AdminSetParent.java | 8 +- 16 files changed, 436 insertions(+), 21 deletions(-) create mode 100644 gerrit-extension-api/src/main/java/com/google/gerrit/extensions/events/ProjectIndexedListener.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexer.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java diff --git a/Documentation/dev-plugins.txt b/Documentation/dev-plugins.txt index ad2c50cdae..9debc2e0ce 100644 --- a/Documentation/dev-plugins.txt +++ b/Documentation/dev-plugins.txt @@ -407,6 +407,10 @@ Update of the account secondary index + Update of the group secondary index +* `com.google.gerrit.server.extensions.events.ProjectIndexedListener`: ++ +Update of the project secondary index + * `com.google.gerrit.httpd.WebLoginListener`: + User login or logout interactively on the Web user interface. diff --git a/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/events/ProjectIndexedListener.java b/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/events/ProjectIndexedListener.java new file mode 100644 index 0000000000..bfbd851bc5 --- /dev/null +++ b/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/events/ProjectIndexedListener.java @@ -0,0 +1,28 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.extensions.events; + +import com.google.gerrit.extensions.annotations.ExtensionPoint; + +/** Notified whenever a project is indexed */ +@ExtensionPoint +public interface ProjectIndexedListener { + /** + * Invoked when a project is indexed + * + * @param name of the project + */ + void onProjectIndexed(String project); +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/receive/ReceiveCommits.java index b7aa416c1f..cff720cd1f 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/receive/ReceiveCommits.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/receive/ReceiveCommits.java @@ -2563,7 +2563,11 @@ class ReceiveCommits { } if (isConfig(cmd)) { logDebug("Reloading project in cache"); - projectCache.evict(project); + try { + projectCache.evict(project); + } catch (IOException e) { + log.warn("Cannot evict from project cache, name key: " + project.getName(), e); + } ProjectState ps = projectCache.get(project.getNameKey()); try { logDebug("Updating project description"); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java index 6854a8772f..8c9a964dd5 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java @@ -45,6 +45,12 @@ import com.google.gerrit.server.index.group.GroupIndexRewriter; import com.google.gerrit.server.index.group.GroupIndexer; import com.google.gerrit.server.index.group.GroupIndexerImpl; import com.google.gerrit.server.index.group.GroupSchemaDefinitions; +import com.google.gerrit.server.index.project.ProjectIndexCollection; +import com.google.gerrit.server.index.project.ProjectIndexDefinition; +import com.google.gerrit.server.index.project.ProjectIndexRewriter; +import com.google.gerrit.server.index.project.ProjectIndexer; +import com.google.gerrit.server.index.project.ProjectIndexerImpl; +import com.google.gerrit.server.index.project.ProjectSchemaDefinitions; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Provides; @@ -70,7 +76,8 @@ public class IndexModule extends LifecycleModule { ImmutableList.>of( AccountSchemaDefinitions.INSTANCE, ChangeSchemaDefinitions.INSTANCE, - GroupSchemaDefinitions.INSTANCE); + GroupSchemaDefinitions.INSTANCE, + ProjectSchemaDefinitions.INSTANCE); /** Type of secondary index. */ public static IndexType getIndexType(Injector injector) { @@ -112,14 +119,22 @@ public class IndexModule extends LifecycleModule { listener().to(GroupIndexCollection.class); factory(GroupIndexerImpl.Factory.class); + bind(ProjectIndexRewriter.class); + bind(ProjectIndexCollection.class); + listener().to(ProjectIndexCollection.class); + factory(ProjectIndexerImpl.Factory.class); + DynamicSet.setOf(binder(), OnlineUpgradeListener.class); } @Provides Collection> getIndexDefinitions( - AccountIndexDefinition accounts, ChangeIndexDefinition changes, GroupIndexDefinition groups) { + AccountIndexDefinition accounts, + ChangeIndexDefinition changes, + GroupIndexDefinition groups, + ProjectIndexDefinition projects) { Collection> result = - ImmutableList.>of(accounts, groups, changes); + ImmutableList.>of(accounts, groups, changes, projects); Set expected = FluentIterable.from(ALL_SCHEMA_DEFS).transform(SchemaDefinitions::getName).toSet(); Set actual = FluentIterable.from(result).transform(IndexDefinition::getName).toSet(); @@ -154,6 +169,13 @@ public class IndexModule extends LifecycleModule { return factory.create(indexes); } + @Provides + @Singleton + ProjectIndexer getProjectIndexer( + ProjectIndexerImpl.Factory factory, ProjectIndexCollection indexes) { + return factory.create(indexes); + } + @Provides @Singleton @IndexExecutor(INTERACTIVE) diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java new file mode 100644 index 0000000000..ede7461f32 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java @@ -0,0 +1,34 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexedQuery; +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.query.DataSource; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryParseException; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.project.ProjectState; + +public class IndexedProjectQuery extends IndexedQuery + implements DataSource { + + public IndexedProjectQuery( + Index index, Predicate pred, QueryOptions opts) + throws QueryParseException { + super(index, pred, opts.convertForBackend()); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java new file mode 100644 index 0000000000..e50d08efd6 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java @@ -0,0 +1,43 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.gerrit.index.IndexRewriter; +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryParseException; +import com.google.gerrit.server.project.ProjectState; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class ProjectIndexRewriter implements IndexRewriter { + private final ProjectIndexCollection indexes; + + @Inject + ProjectIndexRewriter(ProjectIndexCollection indexes) { + this.indexes = indexes; + } + + @Override + public Predicate rewrite(Predicate in, QueryOptions opts) + throws QueryParseException { + ProjectIndex index = indexes.getSearchIndex(); + checkNotNull(index, "no active search index configured for projects"); + return new IndexedProjectQuery(index, in, opts); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexer.java new file mode 100644 index 0000000000..e8a8183b8c --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexer.java @@ -0,0 +1,28 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.gerrit.reviewdb.client.Project; +import java.io.IOException; + +public interface ProjectIndexer { + + /** + * Synchronously index a project. + * + * @param nameKey name key of project to index. + */ + void index(Project.NameKey nameKey) throws IOException; +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java new file mode 100644 index 0000000000..368a056251 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java @@ -0,0 +1,86 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.common.collect.ImmutableSet; +import com.google.gerrit.common.Nullable; +import com.google.gerrit.extensions.events.ProjectIndexedListener; +import com.google.gerrit.extensions.registration.DynamicSet; +import com.google.gerrit.index.Index; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.project.ProjectState; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class ProjectIndexerImpl implements ProjectIndexer { + public interface Factory { + ProjectIndexerImpl create(ProjectIndexCollection indexes); + + ProjectIndexerImpl create(@Nullable ProjectIndex index); + } + + private final ProjectCache projectCache; + private final DynamicSet indexedListener; + private final ProjectIndexCollection indexes; + private final ProjectIndex index; + + @AssistedInject + ProjectIndexerImpl( + ProjectCache projectCache, + DynamicSet indexedListener, + @Assisted ProjectIndexCollection indexes) { + this.projectCache = projectCache; + this.indexedListener = indexedListener; + this.indexes = indexes; + this.index = null; + } + + @AssistedInject + ProjectIndexerImpl( + ProjectCache projectCache, + DynamicSet indexedListener, + @Assisted ProjectIndex index) { + this.projectCache = projectCache; + this.indexedListener = indexedListener; + this.indexes = null; + this.index = index; + } + + @Override + public void index(Project.NameKey nameKey) throws IOException { + for (Index i : getWriteIndexes()) { + i.replace(projectCache.get(nameKey)); + } + fireProjectIndexedEvent(nameKey.get()); + } + + private void fireProjectIndexedEvent(String name) { + for (ProjectIndexedListener listener : indexedListener) { + listener.onProjectIndexed(name); + } + } + + private Collection getWriteIndexes() { + if (indexes != null) { + return indexes.getWriteIndexes(); + } + + return index != null ? Collections.singleton(index) : ImmutableSet.of(); + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/PerRequestProjectControlCache.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/PerRequestProjectControlCache.java index 0f71ac8a37..b68446fe64 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/PerRequestProjectControlCache.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/PerRequestProjectControlCache.java @@ -18,6 +18,7 @@ import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.CurrentUser; import com.google.inject.Inject; import com.google.inject.servlet.RequestScoped; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -48,7 +49,7 @@ public class PerRequestProjectControlCache { return ctl; } - public void evict(Project project) { + public void evict(Project project) throws IOException { projectCache.evict(project); controls.remove(project.getNameKey()); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCache.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCache.java index 65c7315fc8..63052bd5fa 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCache.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCache.java @@ -45,17 +45,27 @@ public interface ProjectCache { */ ProjectState checkedGet(Project.NameKey projectName) throws IOException; - /** Invalidate the cached information about the given project. */ - void evict(Project p); + /** + * Invalidate the cached information about the given project, and triggers reindexing for it + * + * @param p project that is being evicted + * @throws IOException thrown if the reindexing fails + */ + void evict(Project p) throws IOException; - /** Invalidate the cached information about the given project. */ - void evict(Project.NameKey p); + /** + * Invalidate the cached information about the given project, and triggers reindexing for it + * + * @param p the NameKey of the project that is being evicted + * @throws IOException thrown if the reindexing fails + */ + void evict(Project.NameKey p) throws IOException; /** * Remove information about the given project from the cache. It will no longer be returned from * {@link #all()}. */ - void remove(Project p); + void remove(Project p) throws IOException; /** @return sorted iteration of projects. */ Iterable all(); @@ -75,5 +85,5 @@ public interface ProjectCache { Iterable byName(String prefix); /** Notify the cache that a new project was constructed. */ - void onCreateProject(Project.NameKey newProjectName); + void onCreateProject(Project.NameKey newProjectName) throws IOException; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheImpl.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheImpl.java index 6ee143c862..2b31ce3ec9 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheImpl.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheImpl.java @@ -28,8 +28,10 @@ import com.google.gerrit.server.config.AllProjectsName; import com.google.gerrit.server.config.AllUsersName; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.ProjectConfig; +import com.google.gerrit.server.index.project.ProjectIndexer; import com.google.inject.Inject; import com.google.inject.Module; +import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.TypeLiteral; import com.google.inject.internal.UniqueAnnotations; @@ -82,6 +84,7 @@ public class ProjectCacheImpl implements ProjectCache { private final LoadingCache> list; private final Lock listLock; private final ProjectCacheClock clock; + private final Provider indexer; @Inject ProjectCacheImpl( @@ -89,13 +92,15 @@ public class ProjectCacheImpl implements ProjectCache { final AllUsersName allUsersName, @Named(CACHE_NAME) LoadingCache byName, @Named(CACHE_LIST) LoadingCache> list, - ProjectCacheClock clock) { + ProjectCacheClock clock, + Provider indexer) { this.allProjectsName = allProjectsName; this.allUsersName = allUsersName; this.byName = byName; this.list = list; this.listLock = new ReentrantLock(true /* fair */); this.clock = clock; + this.indexer = indexer; } @Override @@ -151,22 +156,20 @@ public class ProjectCacheImpl implements ProjectCache { } @Override - public void evict(Project p) { - if (p != null) { - byName.invalidate(p.getNameKey().get()); - } + public void evict(Project p) throws IOException { + evict(p.getNameKey()); } - /** Invalidate the cached information about the given project. */ @Override - public void evict(Project.NameKey p) { + public void evict(Project.NameKey p) throws IOException { if (p != null) { byName.invalidate(p.get()); } + indexer.get().index(p); } @Override - public void remove(Project p) { + public void remove(Project p) throws IOException { listLock.lock(); try { SortedSet n = Sets.newTreeSet(list.get(ListKey.ALL)); @@ -181,7 +184,7 @@ public class ProjectCacheImpl implements ProjectCache { } @Override - public void onCreateProject(Project.NameKey newProjectName) { + public void onCreateProject(Project.NameKey newProjectName) throws IOException { listLock.lock(); try { SortedSet n = Sets.newTreeSet(list.get(ListKey.ALL)); @@ -192,6 +195,7 @@ public class ProjectCacheImpl implements ProjectCache { } finally { listLock.unlock(); } + indexer.get().index(newProjectName); } @Override diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectControl.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectControl.java index 7a7418c778..1166970e3e 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectControl.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectControl.java @@ -390,6 +390,10 @@ public class ProjectControl { } } + public boolean canRead() { + return !isHidden() && allRefsAreVisible(Collections.emptySet()); + } + ForProject asForProject() { return new ForProjectImpl(); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java new file mode 100644 index 0000000000..07b1722aa9 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java @@ -0,0 +1,48 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.project; + +import com.google.gerrit.index.query.IsVisibleToPredicate; +import com.google.gerrit.server.CurrentUser; +import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.permissions.PermissionBackend; +import com.google.gerrit.server.permissions.ProjectPermission; +import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.query.account.AccountQueryBuilder; +import com.google.gwtorm.server.OrmException; + +public class ProjectIsVisibleToPredicate extends IsVisibleToPredicate { + protected final PermissionBackend permissionBackend; + protected final CurrentUser user; + + public ProjectIsVisibleToPredicate(PermissionBackend permissionBackend, CurrentUser user) { + super(AccountQueryBuilder.FIELD_VISIBLETO, IndexUtils.describe(user)); + this.permissionBackend = permissionBackend; + this.user = user; + } + + @Override + public boolean match(ProjectState projectState) throws OrmException { + return permissionBackend + .user(user) + .project(projectState.getProject().getNameKey()) + .testOrFalse(ProjectPermission.READ); + } + + @Override + public int getCost() { + return 1; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java index f3efdc183d..2457f330c5 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java @@ -14,14 +14,19 @@ package com.google.gerrit.server.query.project; +import com.google.common.primitives.Ints; +import com.google.gerrit.index.query.LimitPredicate; import com.google.gerrit.index.query.Predicate; import com.google.gerrit.index.query.QueryBuilder; +import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.project.ProjectState; import com.google.inject.Inject; /** Parses a query string meant to be applied to project objects. */ public class ProjectQueryBuilder extends QueryBuilder { + public static final String FIELD_LIMIT = "limit"; + private static final QueryBuilder.Definition mydef = new QueryBuilder.Definition<>(ProjectQueryBuilder.class); @@ -34,4 +39,13 @@ public class ProjectQueryBuilder extends QueryBuilder { public Predicate name(String name) { return ProjectPredicates.name(new Project.NameKey(name)); } + + @Operator + public Predicate limit(String query) throws QueryParseException { + Integer limit = Ints.tryParse(query); + if (limit == null) { + throw error("Invalid limit: " + query); + } + return new LimitPredicate<>(FIELD_LIMIT, limit); + } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java new file mode 100644 index 0000000000..234a67b033 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java @@ -0,0 +1,79 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.project; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.gerrit.server.query.project.ProjectQueryBuilder.FIELD_LIMIT; + +import com.google.gerrit.index.IndexConfig; +import com.google.gerrit.index.query.AndSource; +import com.google.gerrit.index.query.IndexPredicate; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryProcessor; +import com.google.gerrit.metrics.MetricMaker; +import com.google.gerrit.server.CurrentUser; +import com.google.gerrit.server.account.AccountLimits; +import com.google.gerrit.server.index.project.ProjectIndexCollection; +import com.google.gerrit.server.index.project.ProjectIndexRewriter; +import com.google.gerrit.server.index.project.ProjectSchemaDefinitions; +import com.google.gerrit.server.permissions.PermissionBackend; +import com.google.gerrit.server.project.ProjectState; +import com.google.inject.Inject; +import com.google.inject.Provider; + +/** + * Query processor for the project index. + * + *

Instances are one-time-use. Other singleton classes should inject a Provider rather than + * holding on to a single instance. + */ +public class ProjectQueryProcessor extends QueryProcessor { + private final PermissionBackend permissionBackend; + private final Provider userProvider; + + static { + // It is assumed that basic rewrites do not touch visibleto predicates. + checkState( + !ProjectIsVisibleToPredicate.class.isAssignableFrom(IndexPredicate.class), + "ProjectQueryProcessor assumes visibleto is not used by the index rewriter."); + } + + @Inject + protected ProjectQueryProcessor( + Provider userProvider, + AccountLimits.Factory limitsFactory, + MetricMaker metricMaker, + IndexConfig indexConfig, + ProjectIndexCollection indexes, + ProjectIndexRewriter rewriter, + PermissionBackend permissionBackend) { + super( + metricMaker, + ProjectSchemaDefinitions.INSTANCE, + indexConfig, + indexes, + rewriter, + FIELD_LIMIT, + () -> limitsFactory.create(userProvider.get()).getQueryLimit()); + this.permissionBackend = permissionBackend; + this.userProvider = userProvider; + } + + @Override + protected Predicate enforceVisibility(Predicate pred) { + return new AndSource<>( + pred, new ProjectIsVisibleToPredicate(permissionBackend, userProvider.get()), start); + } +} diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/AdminSetParent.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/AdminSetParent.java index 0d7fa24e42..6649fcb553 100644 --- a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/AdminSetParent.java +++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/AdminSetParent.java @@ -174,7 +174,13 @@ final class AdminSetParent extends SshCommand { err.append("error: ").append(msg).append("\n"); } - projectCache.evict(nameKey); + try { + projectCache.evict(nameKey); + } catch (IOException e) { + final String msg = "Cannot reindex project: " + name; + log.error(msg, e); + err.append("error: ").append(msg).append("\n"); + } } if (err.length() > 0) { From d8dd1532c806263bd39f0c63a20b22c323db6768 Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Tue, 18 Jul 2017 17:13:52 -0700 Subject: [PATCH 06/11] REST API support for project query Change-Id: I31a253c3cf2263231d6a4be26b5b4fcb655acf94 --- Documentation/rest-api-projects.txt | 59 +++++++++ Documentation/user-search-projects.txt | 36 ++++++ .../server/project/ProjectsCollection.java | 21 ++- .../gerrit/server/project/QueryProjects.java | 120 ++++++++++++++++++ 4 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 Documentation/user-search-projects.txt create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java diff --git a/Documentation/rest-api-projects.txt b/Documentation/rest-api-projects.txt index 9d76d34e51..6e256f16ee 100644 --- a/Documentation/rest-api-projects.txt +++ b/Documentation/rest-api-projects.txt @@ -324,6 +324,65 @@ GET /projects/?type=PERMISSIONS HTTP/1.0 } ---- +[[query-projects]] +=== Query Projects +-- +'GET /projects/?query=' +-- + +Queries projects visible to the caller. The +link:user-search-projects.html#_search_operators[query string] must be +provided by the `query` parameter. The `start` and `limit` parameters +can be used to skip/limit results. + +As result a list of link:#project-info[ProjectInfo] entities is returned. + +.Request +---- + GET /projects/?query=name:test HTTP/1.0 +---- + +.Response +---- + HTTP/1.1 200 OK + Content-Disposition: attachment + Content-Type: application/json; charset=UTF-8 + + )]}' + { + "test": { + "id": "test", + "description": "\u003chtml\u003e is escaped" + } + } +---- + +[[project-query-limit]] +==== Project Limit +The `/projects/?query=` URL also accepts a limit integer in the +`limit` parameter. This limits the results to `limit` projects. + +Query the first 25 projects in project list. +---- + GET /projects/?query=&limit=25 HTTP/1.0 +---- + +The `/projects/` URL also accepts a start integer in the `start` +parameter. The results will skip `start` groups from project list. + +Query 25 projects starting from index 50. +---- + GET /groups/?query=&limit=25&start=50 HTTP/1.0 +---- + +[[project-query-options]] +==== Project Options +Additional fields can be obtained by adding `o` parameters. Each option +requires more lookups and slows down the query response time to the +client so they are generally disabled by default. The supported fields +are described in the context of the link:#project-options[List Projects] +REST endpoint. + [[get-project]] === Get Project -- diff --git a/Documentation/user-search-projects.txt b/Documentation/user-search-projects.txt new file mode 100644 index 0000000000..eff64fa851 --- /dev/null +++ b/Documentation/user-search-projects.txt @@ -0,0 +1,36 @@ += Gerrit Code Review - Searching Projects + +[[search-operators]] +== Search Operators + +Operators act as restrictions on the search. As more operators +are added to the same query string, they further restrict the +returned results. + +[[name]] +name:'NAME':: ++ +Matches projects that have the NAME 'NAME'. + +== Magical Operators + +[[is-visible]] +is:visible:: ++ +Magical internal flag to prove the current user has access to read +the projects and all the refs. This flag is always added to any query. + +[[limit]] +limit:'CNT':: ++ +Limit the returned results to no more than 'CNT' records. This is +automatically set to the page size configured in the current user's +preferences. Including it in a web query may lead to unpredictable +results with regards to pagination. + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] + +SEARCHBOX +--------- diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectsCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectsCollection.java index e0741f00d3..2a79470380 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectsCollection.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectsCollection.java @@ -14,11 +14,14 @@ package com.google.gerrit.server.project; +import com.google.common.collect.ListMultimap; import com.google.gerrit.common.Nullable; import com.google.gerrit.extensions.registration.DynamicMap; import com.google.gerrit.extensions.restapi.AcceptsCreate; import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.IdString; +import com.google.gerrit.extensions.restapi.NeedsParams; import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.extensions.restapi.RestCollection; import com.google.gerrit.extensions.restapi.RestView; @@ -38,32 +41,48 @@ import org.eclipse.jgit.lib.Constants; @Singleton public class ProjectsCollection - implements RestCollection, AcceptsCreate { + implements RestCollection, + AcceptsCreate, + NeedsParams { private final DynamicMap> views; private final Provider list; + private final Provider queryProjects; private final ProjectControl.GenericFactory controlFactory; private final PermissionBackend permissionBackend; private final Provider user; private final CreateProject.Factory createProjectFactory; + private boolean hasQuery; + @Inject ProjectsCollection( DynamicMap> views, Provider list, + Provider queryProjects, ProjectControl.GenericFactory controlFactory, PermissionBackend permissionBackend, CreateProject.Factory factory, Provider user) { this.views = views; this.list = list; + this.queryProjects = queryProjects; this.controlFactory = controlFactory; this.permissionBackend = permissionBackend; this.user = user; this.createProjectFactory = factory; } + @Override + public void setParams(ListMultimap params) throws BadRequestException { + // The --query option is defined in QueryProjects + this.hasQuery = params.containsKey("query"); + } + @Override public RestView list() { + if (hasQuery) { + return queryProjects.get(); + } return list.get().setFormat(OutputFormat.JSON); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java new file mode 100644 index 0000000000..ec3303086c --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java @@ -0,0 +1,120 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.project; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.gerrit.extensions.common.ProjectInfo; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.MethodNotAllowedException; +import com.google.gerrit.extensions.restapi.RestReadView; +import com.google.gerrit.extensions.restapi.TopLevelResource; +import com.google.gerrit.index.query.QueryParseException; +import com.google.gerrit.index.query.QueryResult; +import com.google.gerrit.server.index.project.ProjectIndex; +import com.google.gerrit.server.index.project.ProjectIndexCollection; +import com.google.gerrit.server.query.project.ProjectQueryBuilder; +import com.google.gerrit.server.query.project.ProjectQueryProcessor; +import com.google.gwtorm.server.OrmException; +import com.google.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import org.kohsuke.args4j.Option; + +public class QueryProjects implements RestReadView { + private final ProjectIndexCollection indexes; + private final ProjectQueryBuilder queryBuilder; + private final ProjectQueryProcessor queryProcessor; + private final ProjectJson json; + + private String query; + private int limit; + private int start; + + @Option( + name = "--query", + aliases = {"-q"}, + usage = "project query" + ) + public void setQuery(String query) { + this.query = query; + } + + @Option( + name = "--limit", + aliases = {"-n"}, + metaVar = "CNT", + usage = "maximum number of projects to list" + ) + public void setLimit(int limit) { + this.limit = limit; + } + + @Option( + name = "--start", + aliases = {"-S"}, + metaVar = "CNT", + usage = "number of projects to skip" + ) + public void setStart(int start) { + this.start = start; + } + + @Inject + protected QueryProjects( + ProjectIndexCollection indexes, + ProjectQueryBuilder queryBuilder, + ProjectQueryProcessor queryProcessor, + ProjectJson json) { + this.indexes = indexes; + this.queryBuilder = queryBuilder; + this.queryProcessor = queryProcessor; + this.json = json; + } + + @Override + public List apply(TopLevelResource resource) + throws BadRequestException, MethodNotAllowedException, OrmException { + if (Strings.isNullOrEmpty(query)) { + throw new BadRequestException("missing query field"); + } + + ProjectIndex searchIndex = indexes.getSearchIndex(); + if (searchIndex == null) { + throw new MethodNotAllowedException("no project index"); + } + + if (start != 0) { + queryProcessor.setStart(start); + } + + if (limit != 0) { + queryProcessor.setUserProvidedLimit(limit); + } + + try { + QueryResult result = queryProcessor.query(queryBuilder.parse(query)); + List projects = result.entities(); + + ArrayList projectInfos = Lists.newArrayListWithCapacity(projects.size()); + for (ProjectState projectState : projects) { + projectInfos.add(json.format(projectState.getProject())); + } + return projectInfos; + } catch (QueryParseException e) { + throw new BadRequestException(e.getMessage()); + } + } +} From 36e64ac93aac4e8c4616fe37a6464ce0d69d4f8b Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Thu, 20 Jul 2017 16:59:58 -0700 Subject: [PATCH 07/11] Add query methods to project API Change-Id: I53f3b8611ea9c51c5325e80c17c123eea26b985e --- .../extensions/api/projects/Projects.java | 78 +++++++++++++++++++ .../server/api/projects/ProjectsImpl.java | 37 ++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/api/projects/Projects.java b/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/api/projects/Projects.java index e4a659ce28..02cce3a88f 100644 --- a/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/api/projects/Projects.java +++ b/gerrit-extension-api/src/main/java/com/google/gerrit/extensions/api/projects/Projects.java @@ -58,6 +58,24 @@ public interface Projects { ListRequest list(); + /** + * Query projects. + * + *

Example code: {@code query().withQuery("name:project").get()} + * + * @return API for setting parameters and getting result. + */ + QueryRequest query(); + + /** + * Query projects. + * + *

Shortcut API for {@code query().withQuery(String)}. + * + * @see #query() + */ + QueryRequest query(String query); + abstract class ListRequest { public enum FilterType { CODE, @@ -171,6 +189,56 @@ public interface Projects { } } + /** + * API for setting parameters and getting result. Used for {@code query()}. + * + * @see #query() + */ + abstract class QueryRequest { + private String query; + private int limit; + private int start; + + /** Execute query and returns the matched projects as list. */ + public abstract List get() throws RestApiException; + + /** + * Set query. + * + * @param query needs to be in human-readable form. + */ + public QueryRequest withQuery(String query) { + this.query = query; + return this; + } + + /** + * Set limit for returned list of projects. Optional; server-default is used when not provided. + */ + public QueryRequest withLimit(int limit) { + this.limit = limit; + return this; + } + + /** Set number of projects to skip. Optional; no projects are skipped when not provided. */ + public QueryRequest withStart(int start) { + this.start = start; + return this; + } + + public String getQuery() { + return query; + } + + public int getLimit() { + return limit; + } + + public int getStart() { + return start; + } + } + /** * A default implementation which allows source compatibility when adding new methods to the * interface. @@ -195,5 +263,15 @@ public interface Projects { public ListRequest list() { throw new NotImplementedException(); } + + @Override + public QueryRequest query() { + throw new NotImplementedException(); + } + + @Override + public QueryRequest query(String query) { + throw new NotImplementedException(); + } } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/api/projects/ProjectsImpl.java b/gerrit-server/src/main/java/com/google/gerrit/server/api/projects/ProjectsImpl.java index 702a7e92ee..9490075795 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/api/projects/ProjectsImpl.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/api/projects/ProjectsImpl.java @@ -22,14 +22,18 @@ import com.google.gerrit.extensions.api.projects.Projects; import com.google.gerrit.extensions.common.ProjectInfo; import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.extensions.restapi.TopLevelResource; import com.google.gerrit.extensions.restapi.UnprocessableEntityException; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.gerrit.server.project.ListProjects; import com.google.gerrit.server.project.ListProjects.FilterType; import com.google.gerrit.server.project.ProjectsCollection; +import com.google.gerrit.server.project.QueryProjects; +import com.google.gwtorm.server.OrmException; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; +import java.util.List; import java.util.SortedMap; @Singleton @@ -37,15 +41,18 @@ class ProjectsImpl implements Projects { private final ProjectsCollection projects; private final ProjectApiImpl.Factory api; private final Provider listProvider; + private final Provider queryProvider; @Inject ProjectsImpl( ProjectsCollection projects, ProjectApiImpl.Factory api, - Provider listProvider) { + Provider listProvider, + Provider queryProvider) { this.projects = projects; this.api = api; this.listProvider = listProvider; + this.queryProvider = queryProvider; } @Override @@ -124,4 +131,32 @@ class ProjectsImpl implements Projects { return lp.apply(); } + + @Override + public QueryRequest query() { + return new QueryRequest() { + @Override + public List get() throws RestApiException { + return ProjectsImpl.this.query(this); + } + }; + } + + @Override + public QueryRequest query(String query) { + return query().withQuery(query); + } + + private List query(QueryRequest r) throws RestApiException { + try { + QueryProjects myQueryProjects = queryProvider.get(); + myQueryProjects.setQuery(r.getQuery()); + myQueryProjects.setLimit(r.getLimit()); + myQueryProjects.setStart(r.getStart()); + + return myQueryProjects.apply(TopLevelResource.INSTANCE); + } catch (OrmException e) { + throw new RestApiException("Cannot query projects", e); + } + } } From 428e70e7b390d9224e4f8baff6406df1bcd92154 Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Tue, 18 Jul 2017 16:29:44 -0700 Subject: [PATCH 08/11] Add project index tests for both Lucene and ElasticSearch Change-Id: I38da120ba7eb50c545cb17d1111d915dd39c5a2d --- .../ElasticQueryProjectsTest.java | 65 ++++ .../elasticsearch/ElasticTestUtils.java | 16 + .../server/index/project/ProjectField.java | 2 +- .../project/AbstractQueryProjectsTest.java | 326 ++++++++++++++++++ .../project/LuceneQueryProjectsTest.java | 44 +++ 5 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java create mode 100644 gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java create mode 100644 gerrit-server/src/test/java/com/google/gerrit/server/query/project/LuceneQueryProjectsTest.java diff --git a/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java b/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java new file mode 100644 index 0000000000..4af53e3ed8 --- /dev/null +++ b/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java @@ -0,0 +1,65 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo; +import com.google.gerrit.server.query.project.AbstractQueryProjectsTest; +import com.google.gerrit.testutil.InMemoryModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import java.util.concurrent.ExecutionException; +import org.eclipse.jgit.lib.Config; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class ElasticQueryProjectsTest extends AbstractQueryProjectsTest { + private static ElasticNodeInfo nodeInfo; + + @BeforeClass + public static void startIndexService() throws InterruptedException, ExecutionException { + if (nodeInfo != null) { + // do not start Elasticsearch twice + return; + } + nodeInfo = ElasticTestUtils.startElasticsearchNode(); + ElasticTestUtils.createAllIndexes(nodeInfo); + } + + @AfterClass + public static void stopElasticsearchServer() { + if (nodeInfo != null) { + nodeInfo.node.close(); + nodeInfo.elasticDir.delete(); + nodeInfo = null; + } + } + + @After + public void cleanupIndex() { + if (nodeInfo != null) { + ElasticTestUtils.deleteAllIndexes(nodeInfo); + ElasticTestUtils.createAllIndexes(nodeInfo); + } + } + + @Override + protected Injector createInjector() { + Config elasticsearchConfig = new Config(config); + InMemoryModule.setDefaults(elasticsearchConfig); + ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port); + return Guice.createInjector(new InMemoryModule(elasticsearchConfig, notesMigration)); + } +} diff --git a/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java b/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java index fac10ebe1b..17438c7d8d 100644 --- a/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java +++ b/gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java @@ -20,12 +20,14 @@ import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CHANGES_PREFIX; import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CLOSED_CHANGES; import static com.google.gerrit.elasticsearch.ElasticChangeIndex.OPEN_CHANGES; import static com.google.gerrit.elasticsearch.ElasticGroupIndex.GROUPS_PREFIX; +import static com.google.gerrit.elasticsearch.ElasticProjectIndex.PROJECTS_PREFIX; import com.google.common.base.Strings; import com.google.common.io.Files; import com.google.gerrit.elasticsearch.ElasticAccountIndex.AccountMapping; import com.google.gerrit.elasticsearch.ElasticChangeIndex.ChangeMapping; import com.google.gerrit.elasticsearch.ElasticGroupIndex.GroupMapping; +import com.google.gerrit.elasticsearch.ElasticProjectIndex.ProjectMapping; import com.google.gerrit.index.Schema; import com.google.gerrit.server.account.AccountState; import com.google.gerrit.server.group.InternalGroup; @@ -33,6 +35,8 @@ import com.google.gerrit.server.index.IndexModule.IndexType; import com.google.gerrit.server.index.account.AccountSchemaDefinitions; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; import com.google.gerrit.server.index.group.GroupSchemaDefinitions; +import com.google.gerrit.server.index.project.ProjectSchemaDefinitions; +import com.google.gerrit.server.project.ProjectState; import com.google.gerrit.server.query.change.ChangeData; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; @@ -157,6 +161,18 @@ final class ElasticTestUtils { .addMapping(ElasticGroupIndex.GROUPS, gson.toJson(groupMapping)) .execute() .actionGet(); + + Schema projectSchema = ProjectSchemaDefinitions.INSTANCE.getLatest(); + ProjectMapping projectMapping = new ProjectMapping(projectSchema); + nodeInfo + .node + .client() + .admin() + .indices() + .prepareCreate(String.format("%s%04d", PROJECTS_PREFIX, projectSchema.getVersion())) + .addMapping(ElasticProjectIndex.PROJECTS, gson.toJson(projectMapping)) + .execute() + .actionGet(); } private static String getHttpPort(Node node) throws InterruptedException, ExecutionException { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java index 86156e1be1..7b6bd96904 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java @@ -27,7 +27,7 @@ import com.google.gerrit.server.project.ProjectState; public class ProjectField { public static final FieldDef NAME = - exact("name").build(p -> p.getProject().getName()); + exact("name").stored().build(p -> p.getProject().getName()); public static final FieldDef DESCRIPTION = fullText("description").build(p -> p.getProject().getDescription()); diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java new file mode 100644 index 0000000000..6de89005bc --- /dev/null +++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java @@ -0,0 +1,326 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.project; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toList; + +import com.google.gerrit.extensions.api.GerritApi; +import com.google.gerrit.extensions.api.projects.ProjectInput; +import com.google.gerrit.extensions.api.projects.Projects.QueryRequest; +import com.google.gerrit.extensions.common.AccountInfo; +import com.google.gerrit.extensions.common.ProjectInfo; +import com.google.gerrit.lifecycle.LifecycleManager; +import com.google.gerrit.reviewdb.client.Account; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.server.ReviewDb; +import com.google.gerrit.server.AnonymousUser; +import com.google.gerrit.server.CurrentUser; +import com.google.gerrit.server.IdentifiedUser; +import com.google.gerrit.server.account.AccountCache; +import com.google.gerrit.server.account.AccountManager; +import com.google.gerrit.server.account.Accounts; +import com.google.gerrit.server.account.AccountsUpdate; +import com.google.gerrit.server.account.AuthRequest; +import com.google.gerrit.server.config.AllProjectsName; +import com.google.gerrit.server.query.account.InternalAccountQuery; +import com.google.gerrit.server.schema.SchemaCreator; +import com.google.gerrit.server.util.ManualRequestContext; +import com.google.gerrit.server.util.OneOffRequestContext; +import com.google.gerrit.server.util.RequestContext; +import com.google.gerrit.server.util.ThreadLocalRequestContext; +import com.google.gerrit.testutil.ConfigSuite; +import com.google.gerrit.testutil.GerritServerTests; +import com.google.gerrit.testutil.InMemoryDatabase; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.google.inject.util.Providers; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import org.eclipse.jgit.lib.Config; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public abstract class AbstractQueryProjectsTest extends GerritServerTests { + @ConfigSuite.Default + public static Config defaultConfig() { + Config cfg = new Config(); + cfg.setInt("index", null, "maxPages", 10); + return cfg; + } + + @Inject protected Accounts accounts; + + @Inject protected AccountsUpdate.Server accountsUpdate; + + @Inject protected AccountCache accountCache; + + @Inject protected AccountManager accountManager; + + @Inject protected GerritApi gApi; + + @Inject protected IdentifiedUser.GenericFactory userFactory; + + @Inject private Provider anonymousUser; + + @Inject protected InMemoryDatabase schemaFactory; + + @Inject protected SchemaCreator schemaCreator; + + @Inject protected ThreadLocalRequestContext requestContext; + + @Inject protected OneOffRequestContext oneOffRequestContext; + + @Inject protected InternalAccountQuery internalAccountQuery; + + @Inject protected AllProjectsName allProjects; + + protected LifecycleManager lifecycle; + protected Injector injector; + protected ReviewDb db; + protected AccountInfo currentUserInfo; + protected CurrentUser user; + + protected abstract Injector createInjector(); + + @Before + public void setUpInjector() throws Exception { + lifecycle = new LifecycleManager(); + injector = createInjector(); + lifecycle.add(injector); + injector.injectMembers(this); + lifecycle.start(); + setUpDatabase(); + } + + protected void setUpDatabase() throws Exception { + db = schemaFactory.open(); + schemaCreator.create(db); + + Account.Id userId = createAccount("user", "User", "user@example.com", true); + user = userFactory.create(userId); + requestContext.setContext(newRequestContext(userId)); + currentUserInfo = gApi.accounts().id(userId.get()).get(); + } + + protected RequestContext newRequestContext(Account.Id requestUserId) { + final CurrentUser requestUser = userFactory.create(requestUserId); + return new RequestContext() { + @Override + public CurrentUser getUser() { + return requestUser; + } + + @Override + public Provider getReviewDbProvider() { + return Providers.of(db); + } + }; + } + + protected void setAnonymous() { + requestContext.setContext( + new RequestContext() { + @Override + public CurrentUser getUser() { + return anonymousUser.get(); + } + + @Override + public Provider getReviewDbProvider() { + return Providers.of(db); + } + }); + } + + @After + public void tearDownInjector() { + if (lifecycle != null) { + lifecycle.stop(); + } + requestContext.setContext(null); + if (db != null) { + db.close(); + } + InMemoryDatabase.drop(schemaFactory); + } + + @Test + public void byName() throws Exception { + assertQuery("name:project"); + assertQuery("name:non-existing"); + + ProjectInfo project = createProject(name("project")); + + assertQuery("name:" + project.name, project); + + // only exact match + ProjectInfo projectWithHyphen = createProject(name("project-with-hyphen")); + createProject(name("project-no-match-with-hyphen")); + assertQuery("name:" + projectWithHyphen.name, projectWithHyphen); + } + + @Test + public void withLimit() throws Exception { + ProjectInfo project1 = createProject(name("project1")); + ProjectInfo project2 = createProject(name("project2")); + ProjectInfo project3 = createProject(name("project3")); + + String query = + "name:" + project1.name + " OR name:" + project2.name + " OR name:" + project3.name; + List result = assertQuery(query, project1, project2, project3); + + result = assertQuery(newQuery(query).withLimit(2), result.subList(0, 2)); + } + + @Test + public void withStart() throws Exception { + ProjectInfo project1 = createProject(name("project1")); + ProjectInfo project2 = createProject(name("project2")); + ProjectInfo project3 = createProject(name("project3")); + + String query = + "name:" + project1.name + " OR name:" + project2.name + " OR name:" + project3.name; + List result = assertQuery(query, project1, project2, project3); + + assertQuery(newQuery(query).withStart(1), result.subList(1, 3)); + } + + @Test + public void asAnonymous() throws Exception { + ProjectInfo project = createProject(name("project")); + + setAnonymous(); + assertQuery("name:" + project.name); + } + + private Account.Id createAccount(String username, String fullName, String email, boolean active) + throws Exception { + try (ManualRequestContext ctx = oneOffRequestContext.open()) { + Account.Id id = accountManager.authenticate(AuthRequest.forUser(username)).getAccountId(); + if (email != null) { + accountManager.link(id, AuthRequest.forEmail(email)); + } + accountsUpdate + .create() + .update( + id, + a -> { + a.setFullName(fullName); + a.setPreferredEmail(email); + a.setActive(active); + }); + return id; + } + } + + protected ProjectInfo createProject(String name) throws Exception { + ProjectInput in = new ProjectInput(); + in.name = name; + return gApi.projects().create(in).get(); + } + + protected ProjectInfo createProjectWithDescription(String name, String description) + throws Exception { + ProjectInput in = new ProjectInput(); + in.name = name; + in.description = description; + return gApi.projects().create(in).get(); + } + + protected ProjectInfo getProject(Project.NameKey nameKey) throws Exception { + return gApi.projects().name(nameKey.get()).get(); + } + + protected List assertQuery(Object query, ProjectInfo... projects) throws Exception { + return assertQuery(newQuery(query), projects); + } + + protected List assertQuery(QueryRequest query, ProjectInfo... projects) + throws Exception { + return assertQuery(query, Arrays.asList(projects)); + } + + protected List assertQuery(QueryRequest query, List projects) + throws Exception { + List result = query.get(); + Iterable names = names(result); + assertThat(names) + .named(format(query, result, projects)) + .containsExactlyElementsIn(names(projects)); + return result; + } + + protected QueryRequest newQuery(Object query) { + return gApi.projects().query(query.toString()); + } + + protected String format( + QueryRequest query, List actualProjects, List expectedProjects) { + StringBuilder b = new StringBuilder(); + b.append("query '").append(query.getQuery()).append("' with expected projects "); + b.append(format(expectedProjects)); + b.append(" and result "); + b.append(format(actualProjects)); + return b.toString(); + } + + protected String format(Iterable projects) { + StringBuilder b = new StringBuilder(); + b.append("["); + Iterator it = projects.iterator(); + while (it.hasNext()) { + ProjectInfo p = it.next(); + b.append("{") + .append(p.id) + .append(", ") + .append("name=") + .append(p.name) + .append(", ") + .append("parent=") + .append(p.parent) + .append(", ") + .append("description=") + .append(p.description) + .append("}"); + if (it.hasNext()) { + b.append(", "); + } + } + b.append("]"); + return b.toString(); + } + + protected static Iterable names(ProjectInfo... projects) { + return names(Arrays.asList(projects)); + } + + protected static Iterable names(List projects) { + return projects.stream().map(p -> p.name).collect(toList()); + } + + protected String name(String name) { + if (name == null) { + return null; + } + + return name + "_" + getSanitizedMethodName(); + } +} diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/project/LuceneQueryProjectsTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/project/LuceneQueryProjectsTest.java new file mode 100644 index 0000000000..4a09d87eac --- /dev/null +++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/project/LuceneQueryProjectsTest.java @@ -0,0 +1,44 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.query.project; + +import com.google.gerrit.server.index.project.ProjectSchemaDefinitions; +import com.google.gerrit.testutil.ConfigSuite; +import com.google.gerrit.testutil.InMemoryModule; +import com.google.gerrit.testutil.IndexVersions; +import com.google.inject.Guice; +import com.google.inject.Injector; +import java.util.List; +import java.util.Map; +import org.eclipse.jgit.lib.Config; + +public class LuceneQueryProjectsTest extends AbstractQueryProjectsTest { + @ConfigSuite.Configs + public static Map againstPreviousIndexVersion() { + // the current schema version is already tested by the inherited default config suite + List schemaVersions = + IndexVersions.getWithoutLatest( + com.google.gerrit.server.index.project.ProjectSchemaDefinitions.INSTANCE); + return IndexVersions.asConfigMap( + ProjectSchemaDefinitions.INSTANCE, schemaVersions, "againstIndexVersion", defaultConfig()); + } + + @Override + protected Injector createInjector() { + Config luceneConfig = new Config(config); + InMemoryModule.setDefaults(luceneConfig); + return Guice.createInjector(new InMemoryModule(luceneConfig, notesMigration)); + } +} From bca8232d83ea80c7837d17093c74b7a0d9c4a301 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Fri, 4 Aug 2017 12:01:18 -0400 Subject: [PATCH 09/11] Convert project index to use ProjectData instead of ProjectState With this change, the project index does not depend on the Gerrit server, so that projects under non-Gerrit enabled hosts can be indexed without dependencies on the server. Change-Id: I07beec95d1586432bd078d492111b0f6e57c5dcb --- .../elasticsearch/ElasticProjectIndex.java | 36 +++++++++---------- .../elasticsearch/ElasticTestUtils.java | 4 +-- .../gerrit/lucene/LuceneProjectIndex.java | 36 +++++++++---------- .../gerrit/server/index/DummyIndexModule.java | 4 +-- .../index/project/AllProjectsIndexer.java | 6 ++-- .../index/project/IndexedProjectQuery.java | 8 ++--- .../server/index/project/ProjectField.java | 15 ++++---- .../server/index/project/ProjectIndex.java | 8 ++--- .../index/project/ProjectIndexCollection.java | 4 +-- .../index/project/ProjectIndexDefinition.java | 4 +-- .../index/project/ProjectIndexRewriter.java | 6 ++-- .../index/project/ProjectIndexerImpl.java | 6 ++-- .../project/ProjectSchemaDefinitions.java | 8 ++--- .../gerrit/server/project/ProjectData.java | 36 +++++++++++++++++++ .../gerrit/server/project/ProjectState.java | 4 +++ .../gerrit/server/project/QueryProjects.java | 10 +++--- .../project/ProjectIsVisibleToPredicate.java | 8 ++--- .../query/project/ProjectPredicates.java | 8 ++--- .../query/project/ProjectQueryBuilder.java | 10 +++--- .../query/project/ProjectQueryProcessor.java | 6 ++-- 20 files changed, 133 insertions(+), 94 deletions(-) create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectData.java diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java index c983a44872..780f0236d9 100644 --- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java +++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java @@ -30,7 +30,7 @@ import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.project.ProjectField; import com.google.gerrit.server.index.project.ProjectIndex; import com.google.gerrit.server.project.ProjectCache; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -56,12 +56,12 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ElasticProjectIndex extends AbstractElasticIndex +public class ElasticProjectIndex extends AbstractElasticIndex implements ProjectIndex { static class ProjectMapping { MappingProperties projects; - ProjectMapping(Schema schema) { + ProjectMapping(Schema schema) { this.projects = ElasticMapping.createMapping(schema); } } @@ -80,14 +80,14 @@ public class ElasticProjectIndex extends AbstractElasticIndex projectCache, JestClientBuilder clientBuilder, - @Assisted Schema schema) { + @Assisted Schema schema) { super(cfg, sitePaths, schema, clientBuilder, PROJECTS_PREFIX); this.projectCache = projectCache; this.mapping = new ProjectMapping(schema); } @Override - public void replace(ProjectState projectState) throws IOException { + public void replace(ProjectData projectState) throws IOException { Bulk bulk = new Bulk.Builder() .defaultIndex(indexName) @@ -105,7 +105,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex getSource(Predicate p, QueryOptions opts) + public DataSource getSource(Predicate p, QueryOptions opts) throws QueryParseException { return new QuerySource(p, opts); } @@ -122,15 +122,15 @@ public class ElasticProjectIndex extends AbstractElasticIndex { + private class QuerySource implements DataSource { private final Search search; private final Set fields; - QuerySource(Predicate p, QueryOptions opts) throws QueryParseException { + QuerySource(Predicate p, QueryOptions opts) throws QueryParseException { QueryBuilder qb = queryBuilder.toQueryBuilder(p); fields = IndexUtils.projectFields(opts); SearchSourceBuilder searchSource = @@ -157,9 +157,9 @@ public class ElasticProjectIndex extends AbstractElasticIndex read() throws OrmException { + public ResultSet read() throws OrmException { try { - List results = Collections.emptyList(); + List results = Collections.emptyList(); JestResult result = client.execute(search); if (result.isSucceeded()) { JsonObject obj = result.getJsonObject().getAsJsonObject("hits"); @@ -167,21 +167,21 @@ public class ElasticProjectIndex extends AbstractElasticIndex r = Collections.unmodifiableList(results); - return new ResultSet() { + final List r = Collections.unmodifiableList(results); + return new ResultSet() { @Override - public Iterator iterator() { + public Iterator iterator() { return r.iterator(); } @Override - public List toList() { + public List toList() { return r; } @@ -200,7 +200,7 @@ public class ElasticProjectIndex extends AbstractElasticIndex projectSchema = ProjectSchemaDefinitions.INSTANCE.getLatest(); + Schema projectSchema = ProjectSchemaDefinitions.INSTANCE.getLatest(); ProjectMapping projectMapping = new ProjectMapping(projectSchema); nodeInfo .node diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java index 544649f054..6354f6107d 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneProjectIndex.java @@ -27,7 +27,7 @@ import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.project.ProjectIndex; import com.google.gerrit.server.project.ProjectCache; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.ResultSet; import com.google.inject.Inject; @@ -56,7 +56,7 @@ import org.eclipse.jgit.lib.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LuceneProjectIndex extends AbstractLuceneIndex +public class LuceneProjectIndex extends AbstractLuceneIndex implements ProjectIndex { private static final Logger log = LoggerFactory.getLogger(LuceneProjectIndex.class); @@ -64,7 +64,7 @@ public class LuceneProjectIndex extends AbstractLuceneIndex queryBuilder; + private final QueryBuilder queryBuilder; private final Provider projectCache; - private static Directory dir(Schema schema, Config cfg, SitePaths sitePaths) + private static Directory dir(Schema schema, Config cfg, SitePaths sitePaths) throws IOException { if (LuceneIndexModule.isInMemoryTest(cfg)) { return new RAMDirectory(); @@ -90,7 +90,7 @@ public class LuceneProjectIndex extends AbstractLuceneIndex projectCache, - @Assisted Schema schema) + @Assisted Schema schema) throws IOException { super( schema, @@ -107,7 +107,7 @@ public class LuceneProjectIndex extends AbstractLuceneIndex getSource(Predicate p, QueryOptions opts) + public DataSource getSource(Predicate p, QueryOptions opts) throws QueryParseException { return new QuerySource( opts, @@ -133,7 +133,7 @@ public class LuceneProjectIndex extends AbstractLuceneIndex { + private class QuerySource implements DataSource { private final QueryOptions opts; private final Query query; private final Sort sort; @@ -150,27 +150,27 @@ public class LuceneProjectIndex extends AbstractLuceneIndex read() throws OrmException { + public ResultSet read() throws OrmException { IndexSearcher searcher = null; try { searcher = acquire(); int realLimit = opts.start() + opts.limit(); TopFieldDocs docs = searcher.search(query, realLimit, sort); - List result = new ArrayList<>(docs.scoreDocs.length); + List result = new ArrayList<>(docs.scoreDocs.length); for (int i = opts.start(); i < docs.scoreDocs.length; i++) { ScoreDoc sd = docs.scoreDocs[i]; Document doc = searcher.doc(sd.doc, IndexUtils.projectFields(opts)); - result.add(toProjectState(doc)); + result.add(toProjectData(doc)); } - final List r = Collections.unmodifiableList(result); - return new ResultSet() { + final List r = Collections.unmodifiableList(result); + return new ResultSet() { @Override - public Iterator iterator() { + public Iterator iterator() { return r.iterator(); } @Override - public List toList() { + public List toList() { return r; } @@ -193,8 +193,8 @@ public class LuceneProjectIndex extends AbstractLuceneIndex schema) { + public ProjectIndex create(Schema schema) { throw new UnsupportedOperationException(); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java index 932396559d..a416b0ca3c 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java @@ -24,7 +24,7 @@ import com.google.gerrit.index.SiteIndexer; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.index.IndexExecutor; import com.google.gerrit.server.project.ProjectCache; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.gwtorm.server.OrmException; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -40,7 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton -public class AllProjectsIndexer extends SiteIndexer { +public class AllProjectsIndexer extends SiteIndexer { private static final Logger log = LoggerFactory.getLogger(AllProjectsIndexer.class); @@ -84,7 +84,7 @@ public class AllProjectsIndexer extends SiteIndexer { try { projectCache.evict(name); - index.replace(projectCache.get(name)); + index.replace(projectCache.get(name).toProjectData()); verboseWriter.println("Reindexed " + desc); done.incrementAndGet(); } catch (Exception e) { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java index ede7461f32..41bff05eb6 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/IndexedProjectQuery.java @@ -21,13 +21,13 @@ import com.google.gerrit.index.query.DataSource; import com.google.gerrit.index.query.Predicate; import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; -public class IndexedProjectQuery extends IndexedQuery - implements DataSource { +public class IndexedProjectQuery extends IndexedQuery + implements DataSource { public IndexedProjectQuery( - Index index, Predicate pred, QueryOptions opts) + Index index, Predicate pred, QueryOptions opts) throws QueryParseException { super(index, pred, opts.convertForBackend()); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java index 7b6bd96904..c4f8e9e2ad 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectField.java @@ -21,25 +21,24 @@ import static com.google.gerrit.index.FieldDef.prefix; import com.google.common.collect.Iterables; import com.google.gerrit.index.FieldDef; import com.google.gerrit.index.SchemaUtil; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; /** Index schema for projects. */ public class ProjectField { - public static final FieldDef NAME = + public static final FieldDef NAME = exact("name").stored().build(p -> p.getProject().getName()); - public static final FieldDef DESCRIPTION = + public static final FieldDef DESCRIPTION = fullText("description").build(p -> p.getProject().getDescription()); - public static final FieldDef PARENT_NAME = + public static final FieldDef PARENT_NAME = exact("parent_name").build(p -> p.getProject().getParentName()); - public static final FieldDef> NAME_PART = + public static final FieldDef> NAME_PART = prefix("name_part").buildRepeatable(p -> SchemaUtil.getNameParts(p.getProject().getName())); - public static final FieldDef> ANCESTOR_NAME = + public static final FieldDef> ANCESTOR_NAME = exact("ancestor_name") - .buildRepeatable( - p -> Iterables.transform(p.parents(), parent -> parent.getProject().getName())); + .buildRepeatable(p -> Iterables.transform(p.getAncestors(), n -> n.get())); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java index 80f4fa6184..5fbdf04860 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndex.java @@ -18,16 +18,16 @@ import com.google.gerrit.index.Index; import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.index.query.Predicate; import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.gerrit.server.query.project.ProjectPredicates; -public interface ProjectIndex extends Index { +public interface ProjectIndex extends Index { public interface Factory - extends IndexDefinition.IndexFactory {} + extends IndexDefinition.IndexFactory {} @Override - default Predicate keyPredicate(Project.NameKey nameKey) { + default Predicate keyPredicate(Project.NameKey nameKey) { return ProjectPredicates.name(nameKey); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java index 89f2619823..ee65a08d97 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java @@ -17,13 +17,13 @@ package com.google.gerrit.server.index.project; import com.google.common.annotations.VisibleForTesting; import com.google.gerrit.index.IndexCollection; import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.inject.Inject; import com.google.inject.Singleton; @Singleton public class ProjectIndexCollection - extends IndexCollection { + extends IndexCollection { @Inject @VisibleForTesting diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java index 8cdd28a1d5..301f20998a 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexDefinition.java @@ -17,11 +17,11 @@ package com.google.gerrit.server.index.project; import com.google.gerrit.common.Nullable; import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.inject.Inject; public class ProjectIndexDefinition - extends IndexDefinition { + extends IndexDefinition { @Inject ProjectIndexDefinition( diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java index e50d08efd6..41d882009d 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexRewriter.java @@ -20,12 +20,12 @@ import com.google.gerrit.index.IndexRewriter; import com.google.gerrit.index.QueryOptions; import com.google.gerrit.index.query.Predicate; import com.google.gerrit.index.query.QueryParseException; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.inject.Inject; import com.google.inject.Singleton; @Singleton -public class ProjectIndexRewriter implements IndexRewriter { +public class ProjectIndexRewriter implements IndexRewriter { private final ProjectIndexCollection indexes; @Inject @@ -34,7 +34,7 @@ public class ProjectIndexRewriter implements IndexRewriter { } @Override - public Predicate rewrite(Predicate in, QueryOptions opts) + public Predicate rewrite(Predicate in, QueryOptions opts) throws QueryParseException { ProjectIndex index = indexes.getSearchIndex(); checkNotNull(index, "no active search index configured for projects"); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java index 368a056251..2a51f32fea 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexerImpl.java @@ -21,7 +21,7 @@ import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.index.Index; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.project.ProjectCache; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; import java.io.IOException; @@ -64,8 +64,8 @@ public class ProjectIndexerImpl implements ProjectIndexer { @Override public void index(Project.NameKey nameKey) throws IOException { - for (Index i : getWriteIndexes()) { - i.replace(projectCache.get(nameKey)); + for (Index i : getWriteIndexes()) { + i.replace(projectCache.get(nameKey).toProjectData()); } fireProjectIndexedEvent(nameKey.get()); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java index 4ff6db197a..ccece02e09 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectSchemaDefinitions.java @@ -18,11 +18,11 @@ import static com.google.gerrit.index.SchemaUtil.schema; import com.google.gerrit.index.Schema; import com.google.gerrit.index.SchemaDefinitions; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; -public class ProjectSchemaDefinitions extends SchemaDefinitions { +public class ProjectSchemaDefinitions extends SchemaDefinitions { - static final Schema V1 = + static final Schema V1 = schema( ProjectField.NAME, ProjectField.DESCRIPTION, @@ -33,6 +33,6 @@ public class ProjectSchemaDefinitions extends SchemaDefinitions { public static final ProjectSchemaDefinitions INSTANCE = new ProjectSchemaDefinitions(); private ProjectSchemaDefinitions() { - super("projects", ProjectState.class); + super("projects", ProjectData.class); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectData.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectData.java new file mode 100644 index 0000000000..407529d9cb --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectData.java @@ -0,0 +1,36 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.project; + +import com.google.common.collect.ImmutableList; +import com.google.gerrit.reviewdb.client.Project; + +public class ProjectData { + private final Project project; + private final ImmutableList ancestors; + + public ProjectData(Project project, Iterable ancestors) { + this.project = project; + this.ancestors = ImmutableList.copyOf(ancestors); + } + + public Project getProject() { + return project; + } + + public ImmutableList getAncestors() { + return ancestors; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectState.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectState.java index 3015164dae..bd6386c350 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectState.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectState.java @@ -553,6 +553,10 @@ public class ProjectState { } } + public ProjectData toProjectData() { + return new ProjectData(getProject(), parents().transform(s -> s.getProject().getNameKey())); + } + private String readFile(Path p) throws IOException { return Files.exists(p) ? new String(Files.readAllBytes(p), UTF_8) : null; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java index ec3303086c..998bdb2a5a 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/QueryProjects.java @@ -105,12 +105,12 @@ public class QueryProjects implements RestReadView { } try { - QueryResult result = queryProcessor.query(queryBuilder.parse(query)); - List projects = result.entities(); + QueryResult result = queryProcessor.query(queryBuilder.parse(query)); + List pds = result.entities(); - ArrayList projectInfos = Lists.newArrayListWithCapacity(projects.size()); - for (ProjectState projectState : projects) { - projectInfos.add(json.format(projectState.getProject())); + ArrayList projectInfos = Lists.newArrayListWithCapacity(pds.size()); + for (ProjectData pd : pds) { + projectInfos.add(json.format(pd.getProject())); } return projectInfos; } catch (QueryParseException e) { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java index 07b1722aa9..20032cef65 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectIsVisibleToPredicate.java @@ -19,11 +19,11 @@ import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.ProjectPermission; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.gerrit.server.query.account.AccountQueryBuilder; import com.google.gwtorm.server.OrmException; -public class ProjectIsVisibleToPredicate extends IsVisibleToPredicate { +public class ProjectIsVisibleToPredicate extends IsVisibleToPredicate { protected final PermissionBackend permissionBackend; protected final CurrentUser user; @@ -34,10 +34,10 @@ public class ProjectIsVisibleToPredicate extends IsVisibleToPredicate name(Project.NameKey nameKey) { + public static Predicate name(Project.NameKey nameKey) { return new ProjectPredicate(ProjectField.NAME, nameKey.get()); } - static class ProjectPredicate extends IndexPredicate { - ProjectPredicate(FieldDef def, String value) { + static class ProjectPredicate extends IndexPredicate { + ProjectPredicate(FieldDef def, String value) { super(def, value); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java index 2457f330c5..608ec9a1cf 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java @@ -20,14 +20,14 @@ import com.google.gerrit.index.query.Predicate; import com.google.gerrit.index.query.QueryBuilder; import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.inject.Inject; /** Parses a query string meant to be applied to project objects. */ -public class ProjectQueryBuilder extends QueryBuilder { +public class ProjectQueryBuilder extends QueryBuilder { public static final String FIELD_LIMIT = "limit"; - private static final QueryBuilder.Definition mydef = + private static final QueryBuilder.Definition mydef = new QueryBuilder.Definition<>(ProjectQueryBuilder.class); @Inject @@ -36,12 +36,12 @@ public class ProjectQueryBuilder extends QueryBuilder { } @Operator - public Predicate name(String name) { + public Predicate name(String name) { return ProjectPredicates.name(new Project.NameKey(name)); } @Operator - public Predicate limit(String query) throws QueryParseException { + public Predicate limit(String query) throws QueryParseException { Integer limit = Ints.tryParse(query); if (limit == null) { throw error("Invalid limit: " + query); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java index 234a67b033..1e181e5a9f 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java @@ -29,7 +29,7 @@ import com.google.gerrit.server.index.project.ProjectIndexCollection; import com.google.gerrit.server.index.project.ProjectIndexRewriter; import com.google.gerrit.server.index.project.ProjectSchemaDefinitions; import com.google.gerrit.server.permissions.PermissionBackend; -import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.project.ProjectData; import com.google.inject.Inject; import com.google.inject.Provider; @@ -39,7 +39,7 @@ import com.google.inject.Provider; *

Instances are one-time-use. Other singleton classes should inject a Provider rather than * holding on to a single instance. */ -public class ProjectQueryProcessor extends QueryProcessor { +public class ProjectQueryProcessor extends QueryProcessor { private final PermissionBackend permissionBackend; private final Provider userProvider; @@ -72,7 +72,7 @@ public class ProjectQueryProcessor extends QueryProcessor { } @Override - protected Predicate enforceVisibility(Predicate pred) { + protected Predicate enforceVisibility(Predicate pred) { return new AndSource<>( pred, new ProjectIsVisibleToPredicate(permissionBackend, userProvider.get()), start); } From d992cbcbd0d5676c5032068180002c48d266ce23 Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Fri, 25 Aug 2017 02:01:46 +0000 Subject: [PATCH 10/11] ProjectIndexCollection: Remove unnecessary @Inject Change-Id: I1fbb8196944bb64bc176a95ae5f17b1da9a3f3c3 --- .../gerrit/server/index/project/ProjectIndexCollection.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java index ee65a08d97..eeebfa1784 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/project/ProjectIndexCollection.java @@ -18,14 +18,12 @@ import com.google.common.annotations.VisibleForTesting; import com.google.gerrit.index.IndexCollection; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.project.ProjectData; -import com.google.inject.Inject; import com.google.inject.Singleton; @Singleton public class ProjectIndexCollection extends IndexCollection { - @Inject @VisibleForTesting public ProjectIndexCollection() {} } From a2f3971161abcf2b48df7d12802547cecb5ac21e Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Tue, 3 Oct 2017 17:29:24 -0700 Subject: [PATCH 11/11] Add inname, description and default field to project query builder Change-Id: I671a01ff1280763e7f8ae2eafc9fd11cbb1b5275 --- .../query/project/ProjectPredicates.java | 9 ++++ .../query/project/ProjectQueryBuilder.java | 32 +++++++++++++ .../project/AbstractQueryProjectsTest.java | 46 +++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java index dbaf4a3365..379c564a62 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectPredicates.java @@ -20,12 +20,21 @@ import com.google.gerrit.index.query.Predicate; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.index.project.ProjectField; import com.google.gerrit.server.project.ProjectData; +import java.util.Locale; public class ProjectPredicates { public static Predicate name(Project.NameKey nameKey) { return new ProjectPredicate(ProjectField.NAME, nameKey.get()); } + public static Predicate inname(String name) { + return new ProjectPredicate(ProjectField.NAME_PART, name.toLowerCase(Locale.US)); + } + + public static Predicate description(String description) { + return new ProjectPredicate(ProjectField.DESCRIPTION, description); + } + static class ProjectPredicate extends IndexPredicate { ProjectPredicate(FieldDef def, String value) { super(def, value); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java index 608ec9a1cf..e9e9c0f763 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/query/project/ProjectQueryBuilder.java @@ -14,6 +14,8 @@ package com.google.gerrit.server.query.project; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import com.google.gerrit.index.query.LimitPredicate; import com.google.gerrit.index.query.Predicate; @@ -22,6 +24,7 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.project.ProjectData; import com.google.inject.Inject; +import java.util.List; /** Parses a query string meant to be applied to project objects. */ public class ProjectQueryBuilder extends QueryBuilder { @@ -40,6 +43,35 @@ public class ProjectQueryBuilder extends QueryBuilder { return ProjectPredicates.name(new Project.NameKey(name)); } + @Operator + public Predicate inname(String namePart) { + if (namePart.isEmpty()) { + return name(namePart); + } + return ProjectPredicates.inname(namePart); + } + + @Operator + public Predicate description(String description) throws QueryParseException { + if (Strings.isNullOrEmpty(description)) { + throw error("description operator requires a value"); + } + + return ProjectPredicates.description(description); + } + + @Override + protected Predicate defaultField(String query) throws QueryParseException { + // Adapt the capacity of this list when adding more default predicates. + List> preds = Lists.newArrayListWithCapacity(3); + preds.add(name(query)); + preds.add(inname(query)); + if (!Strings.isNullOrEmpty(query)) { + preds.add(description(query)); + } + return Predicate.or(preds); + } + @Operator public Predicate limit(String query) throws QueryParseException { Integer limit = Ints.tryParse(query); diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java index 6de89005bc..8804b96714 100644 --- a/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java +++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java @@ -17,11 +17,13 @@ package com.google.gerrit.server.query.project; import static com.google.common.truth.Truth.assertThat; import static java.util.stream.Collectors.toList; +import com.google.common.base.CharMatcher; import com.google.gerrit.extensions.api.GerritApi; import com.google.gerrit.extensions.api.projects.ProjectInput; import com.google.gerrit.extensions.api.projects.Projects.QueryRequest; import com.google.gerrit.extensions.common.AccountInfo; import com.google.gerrit.extensions.common.ProjectInfo; +import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.lifecycle.LifecycleManager; import com.google.gerrit.reviewdb.client.Account; import com.google.gerrit.reviewdb.client.Project; @@ -51,6 +53,7 @@ import com.google.inject.util.Providers; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Locale; import org.eclipse.jgit.lib.Config; import org.junit.After; import org.junit.Before; @@ -177,6 +180,49 @@ public abstract class AbstractQueryProjectsTest extends GerritServerTests { assertQuery("name:" + projectWithHyphen.name, projectWithHyphen); } + @Test + public void byInname() throws Exception { + String namePart = getSanitizedMethodName(); + namePart = CharMatcher.is('_').removeFrom(namePart); + + ProjectInfo project1 = createProject(name("project-" + namePart)); + ProjectInfo project2 = createProject(name("project-" + namePart + "-2")); + ProjectInfo project3 = createProject(name("project-" + namePart + "3")); + + assertQuery("inname:" + namePart, project1, project2, project3); + assertQuery("inname:" + namePart.toUpperCase(Locale.US), project1, project2, project3); + assertQuery("inname:" + namePart.toLowerCase(Locale.US), project1, project2, project3); + } + + @Test + public void byDescription() throws Exception { + ProjectInfo project1 = + createProjectWithDescription(name("project1"), "This is a test project."); + ProjectInfo project2 = createProjectWithDescription(name("project2"), "ANOTHER TEST PROJECT."); + createProjectWithDescription(name("project3"), "Maintainers of project foo."); + assertQuery("description:test", project1, project2); + + assertQuery("description:non-existing"); + + exception.expect(BadRequestException.class); + exception.expectMessage("description operator requires a value"); + assertQuery("description:\"\""); + } + + @Test + public void byDefaultField() throws Exception { + ProjectInfo project1 = createProject(name("foo-project")); + ProjectInfo project2 = createProject(name("project2")); + ProjectInfo project3 = + createProjectWithDescription( + name("project3"), + "decription that contains foo and the UUID of project2: " + project2.id); + + assertQuery("non-existing"); + assertQuery("foo", project1, project3); + assertQuery(project2.id, project2, project3); + } + @Test public void withLimit() throws Exception { ProjectInfo project1 = createProject(name("project1"));