List projects by scanning the managed Git directory

Rather than locating projects in the database table, perform a
recursive scan of the managed repository directory and pull up
anything that we find.  This is a first step towards moving all
the project control metadata directly into Git.

Bug: issue 436
Change-Id: I08e0083f14f5c03eb9e49b4895c265d13b828534
Signed-off-by: Shawn O. Pearce <sop@google.com>
This commit is contained in:
Shawn O. Pearce
2011-01-10 10:05:54 -08:00
parent 62defaca1c
commit 13fb707580
13 changed files with 253 additions and 96 deletions

View File

@@ -30,8 +30,9 @@ import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.account.GroupControl;
import com.google.gerrit.server.config.AuthConfig;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.project.NoSuchProjectException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.project.ProjectControl;
import com.google.gwt.user.client.rpc.AsyncCallback;
import com.google.gwtorm.client.OrmException;
import com.google.inject.Inject;
@@ -52,6 +53,7 @@ class SuggestServiceImpl extends BaseServiceImplementation implements
private static final String MAX_SUFFIX = "\u9fa5";
private final AuthConfig authConfig;
private final ProjectControl.Factory projectControlFactory;
private final ProjectCache projectCache;
private final AccountCache accountCache;
private final GroupControl.Factory groupControlFactory;
@@ -62,6 +64,7 @@ class SuggestServiceImpl extends BaseServiceImplementation implements
@Inject
SuggestServiceImpl(final Provider<ReviewDb> schema,
final AuthConfig authConfig,
final ProjectControl.Factory projectControlFactory,
final ProjectCache projectCache, final AccountCache accountCache,
final GroupControl.Factory groupControlFactory,
final IdentifiedUser.GenericFactory userFactory,
@@ -69,6 +72,7 @@ class SuggestServiceImpl extends BaseServiceImplementation implements
@GerritServerConfig final Config cfg) {
super(schema, currentUser);
this.authConfig = authConfig;
this.projectControlFactory = projectControlFactory;
this.projectCache = projectCache;
this.accountCache = accountCache;
this.groupControlFactory = groupControlFactory;
@@ -80,24 +84,24 @@ class SuggestServiceImpl extends BaseServiceImplementation implements
public void suggestProjectNameKey(final String query, final int limit,
final AsyncCallback<List<Project.NameKey>> callback) {
run(callback, new Action<List<Project.NameKey>>() {
public List<Project.NameKey> run(final ReviewDb db) throws OrmException {
final String a = query;
final String b = a + MAX_SUFFIX;
final int max = 10;
final int n = limit <= 0 ? max : Math.min(limit, max);
final int max = 10;
final int n = limit <= 0 ? max : Math.min(limit, max);
final CurrentUser user = currentUser.get();
final List<Project.NameKey> r = new ArrayList<Project.NameKey>();
for (final Project p : db.projects().suggestByName(a, b, n)) {
final ProjectState e = projectCache.get(p.getNameKey());
if (e != null && e.controlFor(user).isVisible()) {
r.add(p.getNameKey());
}
}
return r;
final List<Project.NameKey> r = new ArrayList<Project.NameKey>(n);
for (final Project.NameKey nameKey : projectCache.byName(query)) {
final ProjectControl ctl;
try {
ctl = projectControlFactory.validateFor(nameKey);
} catch (NoSuchProjectException e) {
continue;
}
});
r.add(ctl.getProject().getNameKey());
if (r.size() == n) {
break;
}
}
callback.onSuccess(r);
}
public void suggestAccount(final String query, final Boolean active,

View File

@@ -17,11 +17,9 @@ package com.google.gerrit.httpd.rpc.project;
import com.google.gerrit.httpd.rpc.Handler;
import com.google.gerrit.reviewdb.Project;
import com.google.gerrit.reviewdb.ReviewDb;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.project.NoSuchProjectException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectControl;
import com.google.gwtorm.client.OrmException;
import com.google.inject.Inject;
import java.util.ArrayList;
@@ -35,33 +33,26 @@ class VisibleProjects extends Handler<List<Project>> {
}
private final ProjectControl.Factory projectControlFactory;
private final CurrentUser user;
private final ReviewDb db;
private final ProjectCache projectCache;
@Inject
VisibleProjects(final ProjectControl.Factory projectControlFactory,
final CurrentUser user, final ReviewDb db) {
final ProjectCache projectCache) {
this.projectControlFactory = projectControlFactory;
this.user = user;
this.db = db;
this.projectCache = projectCache;
}
@Override
public List<Project> call() throws OrmException {
final List<Project> result;
if (user.isAdministrator()) {
result = db.projects().all().toList();
} else {
result = new ArrayList<Project>();
for (Project p : db.projects().all().toList()) {
try {
ProjectControl c = projectControlFactory.controlFor(p.getNameKey());
if (c.isVisible() || c.isOwner()) {
result.add(p);
}
} catch (NoSuchProjectException e) {
continue;
public List<Project> call() {
List<Project> result = new ArrayList<Project>();
for (Project.NameKey p : projectCache.all()) {
try {
ProjectControl c = projectControlFactory.controlFor(p);
if (c.isVisible() || c.isOwner()) {
result.add(c.getProject());
}
} catch (NoSuchProjectException e) {
continue;
}
}
Collections.sort(result, new Comparator<Project>() {

View File

@@ -21,7 +21,7 @@ import com.google.gwtorm.client.StringKey;
public final class Project {
/** Project name key */
public static class NameKey extends
StringKey<com.google.gwtorm.client.Key<?>> {
StringKey<com.google.gwtorm.client.Key<?>> implements Comparable<NameKey> {
private static final long serialVersionUID = 1L;
@Column(id = 1)
@@ -44,6 +44,11 @@ public final class Project {
name = newValue;
}
@Override
public int compareTo(NameKey other) {
return get().compareTo(other.get());
}
/** Parse a Project.NameKey out of a string representation. */
public static NameKey parse(final String str) {
final NameKey r = new NameKey();

View File

@@ -26,8 +26,4 @@ public interface ProjectAccess extends Access<Project, Project.NameKey> {
@Query("ORDER BY name")
ResultSet<Project> all() throws OrmException;
@Query("WHERE name.name >= ? AND name.name <= ? ORDER BY name LIMIT ?")
ResultSet<Project> suggestByName(String nameA, String nameB, int limit)
throws OrmException;
}

View File

@@ -21,7 +21,7 @@ import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import java.io.IOException;
import java.util.SortedSet;
/**
* Manages Git repositories for the Gerrit server process.
@@ -58,6 +58,9 @@ public interface GitRepositoryManager {
public abstract Repository createRepository(Project.NameKey name)
throws RepositoryNotFoundException;
/** @return set of all known projects, sorted by natural NameKey order. */
public abstract SortedSet<Project.NameKey> list();
/**
* Read the {@code GIT_DIR/description} file for gitweb.
* <p>

View File

@@ -41,6 +41,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
import java.util.SortedSet;
import java.util.TreeSet;
/** Manages Git repositories stored on the local filesystem. */
@Singleton
@@ -227,4 +230,46 @@ public class LocalDiskRepositoryManager implements GitRepositoryManager {
return false; // is a reasonable name
}
@Override
public SortedSet<Project.NameKey> list() {
SortedSet<Project.NameKey> names = new TreeSet<Project.NameKey>();
scanProjects(basePath, "", names);
return Collections.unmodifiableSortedSet(names);
}
private void scanProjects(final File dir, final String prefix,
final SortedSet<Project.NameKey> names) {
final File[] ls = dir.listFiles();
if (ls == null) {
return;
}
for (File f : ls) {
String fileName = f.getName();
if (FileKey.isGitRepository(f, FS.DETECTED)) {
String projectName;
if (fileName.equals(Constants.DOT_GIT)) {
projectName = prefix.substring(0, prefix.length() - 1);
} else if (fileName.endsWith(Constants.DOT_GIT_EXT)) {
int newLen = fileName.length() - Constants.DOT_GIT_EXT.length();
projectName = prefix + fileName.substring(0, newLen);
} else {
projectName = prefix + fileName;
}
Project.NameKey nameKey = new Project.NameKey(projectName);
if (isUnreasonableName(nameKey)) {
log.warn("Ignoring unreasonably named repository " + f.getAbsolutePath());
} else {
names.add(nameKey);
}
} else if (f.isDirectory()) {
scanProjects(f, prefix + f.getName() + "/", names);
}
}
}
}

View File

@@ -15,10 +15,8 @@
package com.google.gerrit.server.git;
import com.google.gerrit.reviewdb.Project;
import com.google.gerrit.reviewdb.ReviewDb;
import com.google.gerrit.server.config.WildProjectName;
import com.google.gwtorm.client.OrmException;
import com.google.gwtorm.client.SchemaFactory;
import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -37,18 +35,17 @@ public class PushAllProjectsOp extends DefaultQueueOp {
private static final Logger log =
LoggerFactory.getLogger(PushAllProjectsOp.class);
private final SchemaFactory<ReviewDb> schema;
private final ProjectCache projectCache;
private final ReplicationQueue replication;
private final Project.NameKey wildProject;
private final String urlMatch;
@Inject
public PushAllProjectsOp(final WorkQueue wq,
final SchemaFactory<ReviewDb> sf, final ReplicationQueue rq,
@WildProjectName final Project.NameKey wp,
public PushAllProjectsOp(final WorkQueue wq, final ProjectCache projectCache,
final ReplicationQueue rq, @WildProjectName final Project.NameKey wp,
@Assisted @Nullable final String urlMatch) {
super(wq);
this.schema = sf;
this.projectCache = projectCache;
this.replication = rq;
this.wildProject = wp;
this.urlMatch = urlMatch;
@@ -63,17 +60,12 @@ public class PushAllProjectsOp extends DefaultQueueOp {
public void run() {
try {
final ReviewDb db = schema.open();
try {
for (final Project project : db.projects().all()) {
if (!project.getNameKey().equals(wildProject)) {
replication.scheduleFullSync(project.getNameKey(), urlMatch);
}
for (final Project.NameKey nameKey : projectCache.all()) {
if (!nameKey.equals(wildProject)) {
replication.scheduleFullSync(nameKey, urlMatch);
}
} finally {
db.close();
}
} catch (OrmException e) {
} catch (RuntimeException e) {
log.error("Cannot enumerate known projects", e);
}
}

View File

@@ -31,4 +31,18 @@ public interface ProjectCache {
/** Invalidate the cached information about all projects. */
public void evictAll();
/** @return sorted iteration of projects. */
public abstract Iterable<Project.NameKey> all();
/**
* Filter the set of registered project names by common prefix.
*
* @param prefix common prefix.
* @return sorted iteration of projects sharing the same prefix.
*/
public abstract Iterable<Project.NameKey> byName(String prefix);
/** Notify the cache that a new project was constructed. */
public void onCreateProject(Project.NameKey newProjectName);
}

View File

@@ -20,6 +20,7 @@ import com.google.gerrit.reviewdb.ReviewDb;
import com.google.gerrit.server.cache.Cache;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.cache.EntryCreator;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gwtorm.client.SchemaFactory;
import com.google.inject.Inject;
import com.google.inject.Module;
@@ -29,19 +30,31 @@ import com.google.inject.name.Named;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/** Cache of project information, including access rights. */
@Singleton
public class ProjectCacheImpl implements ProjectCache {
private static final String CACHE_NAME = "projects";
private static final String CACHE_LIST = "project_list";
public static Module module() {
return new CacheModule() {
@Override
protected void configure() {
final TypeLiteral<Cache<Project.NameKey, ProjectState>> type =
final TypeLiteral<Cache<Project.NameKey, ProjectState>> nameType =
new TypeLiteral<Cache<Project.NameKey, ProjectState>>() {};
core(type, CACHE_NAME).populateWith(Loader.class);
core(nameType, CACHE_NAME).populateWith(Loader.class);
final TypeLiteral<Cache<ListKey, SortedSet<Project.NameKey>>> listType =
new TypeLiteral<Cache<ListKey, SortedSet<Project.NameKey>>>() {};
core(listType, CACHE_LIST).populateWith(Lister.class);
bind(ProjectCacheImpl.class);
bind(ProjectCache.class).to(ProjectCacheImpl.class);
}
@@ -49,11 +62,16 @@ public class ProjectCacheImpl implements ProjectCache {
}
private final Cache<Project.NameKey, ProjectState> byName;
private final Cache<ListKey,SortedSet<Project.NameKey>> list;
private final Lock listLock;
@Inject
ProjectCacheImpl(
@Named(CACHE_NAME) final Cache<Project.NameKey, ProjectState> byName) {
@Named(CACHE_NAME) final Cache<Project.NameKey, ProjectState> byName,
@Named(CACHE_LIST) final Cache<ListKey, SortedSet<Project.NameKey>> list) {
this.byName = byName;
this.list = list;
this.listLock = new ReentrantLock(true /* fair */);
}
/**
@@ -78,6 +96,74 @@ public class ProjectCacheImpl implements ProjectCache {
byName.removeAll();
}
@Override
public void onCreateProject(Project.NameKey newProjectName) {
listLock.lock();
try {
SortedSet<Project.NameKey> n = list.get(ListKey.ALL);
n = new TreeSet<Project.NameKey>(n);
n.add(newProjectName);
list.put(ListKey.ALL, Collections.unmodifiableSortedSet(n));
} finally {
listLock.unlock();
}
}
@Override
public Iterable<Project.NameKey> all() {
return list.get(ListKey.ALL);
}
@Override
public Iterable<Project.NameKey> byName(final String pfx) {
return new Iterable<Project.NameKey>() {
@Override
public Iterator<Project.NameKey> iterator() {
return new Iterator<Project.NameKey>() {
private Project.NameKey next;
private Iterator<Project.NameKey> itr =
list.get(ListKey.ALL).tailSet(new Project.NameKey(pfx)).iterator();
@Override
public boolean hasNext() {
if (next != null) {
return true;
}
if (!itr.hasNext()) {
return false;
}
Project.NameKey r = itr.next();
if (r.get().startsWith(pfx)) {
next = r;
return true;
} else {
itr = Collections.<Project.NameKey> emptyList().iterator();
return false;
}
}
@Override
public Project.NameKey next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
Project.NameKey r = next;
next = null;
return r;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
};
}
static class Loader extends EntryCreator<Project.NameKey, ProjectState> {
private final ProjectState.Factory projectStateFactory;
private final SchemaFactory<ReviewDb> schema;
@@ -107,4 +193,25 @@ public class ProjectCacheImpl implements ProjectCache {
}
}
}
static class ListKey {
static final ListKey ALL = new ListKey();
private ListKey() {
}
}
static class Lister extends EntryCreator<ListKey, SortedSet<Project.NameKey>> {
private final GitRepositoryManager mgr;
@Inject
Lister(GitRepositoryManager mgr) {
this.mgr = mgr;
}
@Override
public SortedSet<Project.NameKey> createEntry(ListKey key) throws Exception {
return mgr.list();
}
}
}

View File

@@ -30,6 +30,7 @@ import com.google.gerrit.server.config.WildProjectName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.query.IntPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryBuilder;
@@ -107,6 +108,7 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
final Project.NameKey wildProjectName;
final PatchListCache patchListCache;
final GitRepositoryManager repoManager;
final ProjectCache projectCache;
@Inject
Arguments(Provider<ReviewDb> dbProvider,
@@ -118,7 +120,8 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
AuthConfig authConfig, ApprovalTypes approvalTypes,
@WildProjectName Project.NameKey wildProjectName,
PatchListCache patchListCache,
GitRepositoryManager repoManager) {
GitRepositoryManager repoManager,
ProjectCache projectCache) {
this.dbProvider = dbProvider;
this.rewriter = rewriter;
this.userFactory = userFactory;
@@ -131,6 +134,7 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
this.wildProjectName = wildProjectName;
this.patchListCache = patchListCache;
this.repoManager = repoManager;
this.projectCache = projectCache;
}
}
@@ -508,23 +512,19 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
// Try to match a project name by substring query.
final List<ProjectPredicate> predicate =
new ArrayList<ProjectPredicate>();
try {
for (final Project p : args.dbProvider.get().projects().all()) {
if (p.getName().toLowerCase().contains(query.toLowerCase())) {
predicate.add(new ProjectPredicate(args.dbProvider, p.getName()));
}
for (Project.NameKey name : args.projectCache.all()) {
if (name.get().toLowerCase().contains(query.toLowerCase())) {
predicate.add(new ProjectPredicate(args.dbProvider, name.get()));
}
}
// If two or more projects contains "query" as substring create an
// OrPredicate holding predicates for all these projects, otherwise if
// only one contains that, return only that one predicate by itself.
if (predicate.size() == 1) {
return predicate.get(0);
} else if (predicate.size() > 1) {
return Predicate.or(predicate);
}
} catch (OrmException e) {
throw error("Cannot lookup project.", e);
// If two or more projects contains "query" as substring create an
// OrPredicate holding predicates for all these projects, otherwise if
// only one contains that, return only that one predicate by itself.
if (predicate.size() == 1) {
return predicate.get(0);
} else if (predicate.size() > 1) {
return Predicate.or(predicate);
}
throw error("Unsupported query:" + query);

View File

@@ -38,7 +38,8 @@ public class ChangeQueryRewriter extends QueryRewriter<ChangeData> {
new ChangeQueryBuilder.Arguments( //
new InvalidProvider<ReviewDb>(), //
new InvalidProvider<ChangeQueryRewriter>(), //
null, null, null, null, null, null, null, null, null, null), null));
null, null, null, null, null, null, null, //
null, null, null, null), null));
private final Provider<ReviewDb> dbProvider;

View File

@@ -27,6 +27,7 @@ import com.google.gerrit.server.config.ProjectCreatorGroups;
import com.google.gerrit.server.config.ProjectOwnerGroups;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.ReplicationQueue;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectControl;
import com.google.gerrit.sshd.BaseCommand;
import com.google.gwtorm.client.OrmException;
@@ -101,6 +102,9 @@ final class CreateProject extends BaseCommand {
@Inject
private GitRepositoryManager repoManager;
@Inject
private ProjectCache projectCache;
@Inject
@ProjectCreatorGroups
private Set<AccountGroup.Id> projectCreatorGroups;
@@ -223,6 +227,7 @@ final class CreateProject extends BaseCommand {
}
db.projects().insert(Collections.singleton(newProject));
projectCache.onCreateProject(newProject.getNameKey());
}
private void validateParameters() throws Failure {

View File

@@ -15,7 +15,6 @@
package com.google.gerrit.sshd.commands;
import com.google.gerrit.reviewdb.Project;
import com.google.gerrit.reviewdb.ReviewDb;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.config.WildProjectName;
import com.google.gerrit.server.git.GitRepositoryManager;
@@ -23,7 +22,6 @@ import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectControl;
import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.sshd.BaseCommand;
import com.google.gwtorm.client.OrmException;
import com.google.inject.Inject;
import org.apache.sshd.server.Environment;
@@ -43,9 +41,6 @@ final class ListProjects extends BaseCommand {
private static final String DEFAULT_TAB_SEPARATOR = "|";
private static final String NOT_VISIBLE_PROJECT = "(x)";
@Inject
private ReviewDb db;
@Inject
private IdentifiedUser currentUser;
@@ -93,14 +88,14 @@ final class ListProjects extends BaseCommand {
}
try {
for (final Project p : db.projects().all()) {
if (p.getNameKey().equals(wildProject)) {
for (final Project.NameKey projectName : projectCache.all()) {
if (projectName.equals(wildProject)) {
// This project "doesn't exist". At least not as a repository.
//
continue;
}
final ProjectState e = projectCache.get(p.getNameKey());
final ProjectState e = projectCache.get(projectName);
if (e == null) {
// If we can't get it from the cache, pretend its not present.
//
@@ -118,7 +113,7 @@ final class ListProjects extends BaseCommand {
}
if (showBranch != null) {
final Ref ref = getBranchRef(p.getNameKey());
final Ref ref = getBranchRef(projectName);
if (ref == null || ref.getObjectId() == null
|| !pctl.controlForRef(ref.getLeaf().getName()).isVisible()) {
// No branch, or the user can't see this branch, so skip it.
@@ -130,9 +125,10 @@ final class ListProjects extends BaseCommand {
stdout.print(' ');
}
stdout.print(p.getName() + "\n");
stdout.print(projectName.get() + "\n");
} else {
treeMap.put(p.getName(), new TreeNode(p, pctl.isVisible()));
treeMap.put(projectName.get(),
new TreeNode(pctl.getProject(), pctl.isVisible()));
}
}
@@ -144,7 +140,7 @@ final class ListProjects extends BaseCommand {
for (final TreeNode key : treeMap.values()) {
final String parentName = key.getParentName();
if (parentName != null) {
final TreeNode node = treeMap.get((String)parentName);
final TreeNode node = treeMap.get(parentName);
if (node != null) {
node.addChild(key);
} else {
@@ -161,8 +157,6 @@ final class ListProjects extends BaseCommand {
printElement(stdout, fakeRoot, -1, false, sortedNodes.get(sortedNodes.size() - 1));
stdout.flush();
}
} catch (OrmException e) {
throw new Failure(1, "fatal: database error", e);
} finally {
stdout.flush();
}