Support multiple git repositories locations

Gerrit was supporting only one location for git repositories configured
with gerrit.basePath. Now supports alternate location for the
gerrit.basePath that can be configured with repository.name.basePath.

The motivation for this change is to allow to host projects on different
storage. This could be required for performance reasons, e.g. use a high
performance storage for a heavily used repository.

In the following example, all the repositories go in
/my/gerrit/site/git, except the ones with a name starting with
"organizationA", which go in /other/volume/git. If a new project is
created and its name starts with "organizationA", then it will be
created and used from /other/volume/git.

[gerrit]
  basePath = /my/gerrit/site/git
[repository "organizationA*"]
  basePath = /other/volume/git

An alternative to this change is using soft links in the gerrit.basePath
folder to allow putting some repositories on another storage. This
solution is not viable if gerrit.basePath folder is not on local
storage. For example, if the gerrit site, including the basePath for
repositories, are on NFS, then using a soft link on the NFS to link to a
high performance storage would not result in the expected performance
improvement. That is so because any repositories stored on the high
performance storage would be used through the NFS storage.

Change-Id: I1212d0226bfbb51ee92be4902c3658078d56b841

Documentation/config-gerrit.txt

@@ -3190,6 +3190,17 @@ named `project/plugins/a` would be `CHERRY_PICK`.
 the previous example, all properties will be used from `project/plugins/\*`
 section and no properties will be inherited nor overridden from `project/*`.
 
+[[repository.name.basePath]]repository.<name>.basePath::
++
+Alternate to <<gerrit.basePath,gerrit.basePath>>. The repository will be created
+and used from this location instead: ${alternateBasePath}/${projectName}.git.
++
+If configuring the basePath for an existing project in gerrit, make sure to stop
+gerrit, move the repository in the alternate basePath, configure basePath for
+this repository and then start Gerrit.
++
+Path must be absolute.
+
 [[repository.name.defaultSubmitType]]repository.<name>.defaultSubmitType::
 +
 The default submit type for newly created projects. Supported values

gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/SiteProgram.java

@@ -28,7 +28,7 @@ import com.google.gerrit.metrics.dropwizard.DropWizardMetricMaker;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.GerritServerConfigModule;
 import com.google.gerrit.server.config.SitePath;
-import com.google.gerrit.server.git.LocalDiskRepositoryManager;
+import com.google.gerrit.server.git.GitRepositoryManagerModule;
 import com.google.gerrit.server.notedb.ConfigNotesMigration;
 import com.google.gerrit.server.schema.DataSourceModule;
 import com.google.gerrit.server.schema.DataSourceProvider;
@@ -175,7 +175,7 @@ public abstract class SiteProgram extends AbstractProgram {
     });
     modules.add(new DatabaseModule());
     modules.add(new SchemaModule());
-    modules.add(new LocalDiskRepositoryManager.Module());
+    modules.add(cfgInjector.getInstance(GitRepositoryManagerModule.class));
     modules.add(new ConfigNotesMigration.Module());
 
     try {

gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java

@@ -223,7 +223,6 @@ public class GerritGlobalModule extends FactoryModule {
         .toProvider(AccountVisibilityProvider.class)
         .in(SINGLETON);
     factory(ProjectOwnerGroupsProvider.Factory.class);
-    bind(RepositoryConfig.class);
 
     bind(AuthBackend.class).to(UniversalAuthBackend.class).in(SINGLETON);
     DynamicSet.setOf(binder(), AuthBackend.class);

gerrit-server/src/main/java/com/google/gerrit/server/config/RepositoryConfig.java

@@ -21,12 +21,18 @@ import com.google.inject.Singleton;
 
 import org.eclipse.jgit.lib.Config;
 
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
 @Singleton
 public class RepositoryConfig {
 
   static final String SECTION_NAME = "repository";
   static final String OWNER_GROUP_NAME = "ownerGroup";
   static final String DEFAULT_SUBMIT_TYPE_NAME = "defaultSubmitType";
+  static final String BASE_PATH_NAME = "basePath";
 
   private final Config cfg;
 
@@ -45,6 +51,23 @@ public class RepositoryConfig {
         OWNER_GROUP_NAME);
   }
 
+  public Path getBasePath(Project.NameKey project) {
+    String basePath = cfg.getString(SECTION_NAME, findSubSection(project.get()),
+        BASE_PATH_NAME);
+    return basePath != null ? Paths.get(basePath) : null;
+  }
+
+  public List<Path> getAllBasePaths() {
+    List<Path> basePaths = new ArrayList<>();
+    for (String subSection : cfg.getSubsections(SECTION_NAME)) {
+      String basePath = cfg.getString(SECTION_NAME, subSection, BASE_PATH_NAME);
+      if (basePath != null) {
+        basePaths.add(Paths.get(basePath));
+      }
+    }
+    return basePaths;
+  }
+
   /**
    * Find the subSection to get repository configuration from.
    * <p>

gerrit-server/src/main/java/com/google/gerrit/server/git/GitRepositoryManagerModule.java

@@ -0,0 +1,38 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.RepositoryConfig;
+import com.google.inject.Inject;
+
+public class GitRepositoryManagerModule extends LifecycleModule {
+
+  private final RepositoryConfig repoConfig;
+
+  @Inject
+  public GitRepositoryManagerModule(RepositoryConfig repoConfig) {
+    this.repoConfig = repoConfig;
+  }
+
+  @Override
+  protected void configure() {
+    if (repoConfig.getAllBasePaths().isEmpty()) {
+      install(new LocalDiskRepositoryManager.Module());
+    } else {
+      install(new MultiBaseLocalDiskRepositoryManager.Module());
+    }
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/git/MultiBaseLocalDiskRepositoryManager.java

@@ -0,0 +1,78 @@
+//Copyright (C) 2015 The Android Open Source Project
+//
+//Licensed under the Apache License, Version 2.0 (the "License");
+//you may not use this file except in compliance with the License.
+//You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing, software
+//distributed under the License is distributed on an "AS IS" BASIS,
+//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//See the License for the specific language governing permissions and
+//limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.reviewdb.client.Project.NameKey;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.RepositoryConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.inject.Inject;
+
+import org.eclipse.jgit.lib.Config;
+
+import java.nio.file.Path;
+
+public class MultiBaseLocalDiskRepositoryManager extends
+    LocalDiskRepositoryManager {
+
+  public static class Module extends LifecycleModule {
+    @Override
+    protected void configure() {
+      bind(GitRepositoryManager.class).to(
+          MultiBaseLocalDiskRepositoryManager.class);
+      bind(LocalDiskRepositoryManager.class).to(
+          MultiBaseLocalDiskRepositoryManager.class);
+      listener().to(MultiBaseLocalDiskRepositoryManager.class);
+      listener().to(MultiBaseLocalDiskRepositoryManager.Lifecycle.class);
+    }
+  }
+
+  private final RepositoryConfig config;
+
+  @Inject
+  MultiBaseLocalDiskRepositoryManager(SitePaths site,
+      @GerritServerConfig Config cfg,
+      NotesMigration notesMigration,
+      RepositoryConfig config) {
+    super(site, cfg, notesMigration);
+    this.config = config;
+
+    for (Path alternateBasePath : config.getAllBasePaths()) {
+      checkState(alternateBasePath.isAbsolute(),
+          "repository.<name>.basePath must be absolute: %s", alternateBasePath);
+    }
+  }
+
+  @Override
+  public Path getBasePath(NameKey name) {
+    Path alternateBasePath = config.getBasePath(name);
+    return alternateBasePath != null
+        ? alternateBasePath
+        : super.getBasePath(name);
+  }
+
+  @Override
+  protected void scanProjects(ProjectVisitor visitor) {
+    super.scanProjects(visitor);
+    for (Path path : config.getAllBasePaths()) {
+      visitor.setStartFolder(path);
+      super.scanProjects(visitor);
+    }
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/config/RepositoryConfigTest.java

@@ -24,6 +24,9 @@ import org.eclipse.jgit.lib.Config;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.List;
 
 public class RepositoryConfigTest {
@@ -144,4 +147,67 @@ public class RepositoryConfigTest {
     cfg.setStringList(RepositoryConfig.SECTION_NAME, projectFilter,
         RepositoryConfig.OWNER_GROUP_NAME, ownerGroups);
   }
+
+  @Test
+  public void testBasePathWhenNotConfigured() {
+    assertThat((Object)repoCfg.getBasePath(new NameKey("someProject"))).isNull();
+  }
+
+  @Test
+  public void testBasePathForStarFilter() {
+    String basePath = "/someAbsolutePath/someDirectory";
+    configureBasePath("*", basePath);
+    assertThat(repoCfg.getBasePath(new NameKey("someProject")).toString())
+        .isEqualTo(basePath);
+  }
+
+  @Test
+  public void testBasePathForSpecificFilter() {
+    String basePath = "/someAbsolutePath/someDirectory";
+    configureBasePath("someProject", basePath);
+    assertThat((Object) repoCfg.getBasePath(new NameKey("someOtherProject")))
+        .isNull();
+    assertThat(repoCfg.getBasePath(new NameKey("someProject")).toString())
+        .isEqualTo(basePath);
+  }
+
+  @Test
+  public void testBasePathForStartWithFilter() {
+    String basePath1 = "/someAbsolutePath1/someDirectory";
+    String basePath2 = "someRelativeDirectory2";
+    String basePath3 = "/someAbsolutePath3/someDirectory";
+    String basePath4 = "/someAbsolutePath4/someDirectory";
+
+    configureBasePath("pro*", basePath1);
+    configureBasePath("project/project/*", basePath2);
+    configureBasePath("project/*", basePath3);
+    configureBasePath("*", basePath4);
+
+    assertThat(repoCfg.getBasePath(new NameKey("project1")).toString())
+        .isEqualTo(basePath1);
+    assertThat(repoCfg.getBasePath(new NameKey("project/project/someProject"))
+        .toString()).isEqualTo(basePath2);
+    assertThat(
+        repoCfg.getBasePath(new NameKey("project/someProject")).toString())
+            .isEqualTo(basePath3);
+    assertThat(repoCfg.getBasePath(new NameKey("someProject")).toString())
+        .isEqualTo(basePath4);
+  }
+
+  @Test
+  public void testAllBasePath() {
+    List<Path> allBasePaths = Arrays.asList(Paths.get("/someBasePath1"),
+        Paths.get("/someBasePath2"), Paths.get("/someBasePath2"));
+
+    configureBasePath("*", allBasePaths.get(0).toString());
+    configureBasePath("project/*", allBasePaths.get(1).toString());
+    configureBasePath("project/project/*", allBasePaths.get(2).toString());
+
+    assertThat(repoCfg.getAllBasePaths()).isEqualTo(allBasePaths);
+  }
+
+  private void configureBasePath(String projectFilter, String basePath) {
+    cfg.setString(RepositoryConfig.SECTION_NAME, projectFilter,
+        RepositoryConfig.BASE_PATH_NAME, basePath);
+  }
 }

gerrit-server/src/test/java/com/google/gerrit/server/git/MultiBaseLocalDiskRepositoryManagerTest.java

@@ -0,0 +1,194 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.config.RepositoryConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.gerrit.testutil.TempFileUtil;
+import com.google.gwtorm.client.KeyUtil;
+import com.google.gwtorm.server.StandardKeyEncoder;
+
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.RepositoryCache;
+import org.eclipse.jgit.lib.RepositoryCache.FileKey;
+import org.eclipse.jgit.util.FS;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.SortedSet;
+
+public class MultiBaseLocalDiskRepositoryManagerTest {
+
+  static {
+    KeyUtil.setEncoderImpl(new StandardKeyEncoder());
+  }
+
+  private Config cfg;
+  private SitePaths site;
+  private MultiBaseLocalDiskRepositoryManager repoManager;
+  private RepositoryConfig configMock;
+
+  @Before
+  public void setUp() throws IOException {
+    site = new SitePaths(TempFileUtil.createTempDirectory().toPath());
+    site.resolve("git").toFile().mkdir();
+    cfg = new Config();
+    cfg.setString("gerrit", null, "basePath", "git");
+    configMock = createNiceMock(RepositoryConfig.class);
+    expect(configMock.getAllBasePaths()).andReturn(new ArrayList<Path>()).anyTimes();
+    replay(configMock);
+    NotesMigration notesMigrationMock = createNiceMock(NotesMigration.class);
+    replay(notesMigrationMock);
+    repoManager =
+        new MultiBaseLocalDiskRepositoryManager(site, cfg,
+            notesMigrationMock, configMock);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    TempFileUtil.cleanup();
+  }
+
+  @Test
+  public void testDefaultRepositoryLocation()
+      throws RepositoryCaseMismatchException, RepositoryNotFoundException {
+    Project.NameKey someProjectKey = new Project.NameKey("someProject");
+    Repository repo = repoManager.createRepository(someProjectKey);
+    assertThat(repo.getDirectory()).isNotNull();
+    assertThat(repo.getDirectory().exists()).isTrue();
+    assertThat(repo.getDirectory().getParent()).isEqualTo(
+        repoManager.getBasePath(someProjectKey).toAbsolutePath().toString());
+
+    repo = repoManager.openRepository(someProjectKey);
+    assertThat(repo.getDirectory()).isNotNull();
+    assertThat(repo.getDirectory().exists()).isTrue();
+    assertThat(repo.getDirectory().getParent()).isEqualTo(
+        repoManager.getBasePath(someProjectKey).toAbsolutePath().toString());
+
+    assertThat(
+        repoManager.getBasePath(someProjectKey).toAbsolutePath().toString())
+        .isEqualTo(
+            repoManager.getBasePath(someProjectKey).toAbsolutePath().toString());
+
+    SortedSet<Project.NameKey> repoList = repoManager.list();
+    assertThat(repoList.size()).isEqualTo(1);
+    assertThat(repoList.toArray(new Project.NameKey[repoList.size()]))
+        .isEqualTo(new Project.NameKey[] {someProjectKey});
+  }
+
+  @Test
+  public void testAlternateRepositoryLocation() throws IOException {
+    Path alternateBasePath = TempFileUtil.createTempDirectory().toPath();
+    Project.NameKey someProjectKey = new Project.NameKey("someProject");
+    reset(configMock);
+    expect(configMock.getBasePath(someProjectKey)).andReturn(alternateBasePath)
+        .anyTimes();
+    expect(configMock.getAllBasePaths())
+        .andReturn(Arrays.asList(alternateBasePath)).anyTimes();
+    replay(configMock);
+
+    Repository repo = repoManager.createRepository(someProjectKey);
+    assertThat(repo.getDirectory()).isNotNull();
+    assertThat(repo.getDirectory().exists()).isTrue();
+    assertThat(repo.getDirectory().getParent())
+        .isEqualTo(alternateBasePath.toString());
+
+    repo = repoManager.openRepository(someProjectKey);
+    assertThat(repo.getDirectory()).isNotNull();
+    assertThat(repo.getDirectory().exists()).isTrue();
+    assertThat(repo.getDirectory().getParent())
+        .isEqualTo(alternateBasePath.toString());
+
+    assertThat(
+        repoManager.getBasePath(someProjectKey).toAbsolutePath().toString())
+            .isEqualTo(alternateBasePath.toString());
+
+    SortedSet<Project.NameKey> repoList = repoManager.list();
+    assertThat(repoList.size()).isEqualTo(1);
+    assertThat(repoList.toArray(new Project.NameKey[repoList.size()]))
+        .isEqualTo(new Project.NameKey[] {someProjectKey});
+  }
+
+  @Test
+  public void testListReturnRepoFromProperLocation() throws IOException {
+    Project.NameKey basePathProject = new Project.NameKey("basePathProject");
+    Project.NameKey altPathProject = new Project.NameKey("altPathProject");
+    Project.NameKey misplacedProject1 =
+        new Project.NameKey("misplacedProject1");
+    Project.NameKey misplacedProject2 =
+        new Project.NameKey("misplacedProject2");
+
+    Path alternateBasePath = TempFileUtil.createTempDirectory().toPath();
+
+    reset(configMock);
+    expect(configMock.getBasePath(altPathProject)).andReturn(alternateBasePath)
+        .anyTimes();
+    expect(configMock.getBasePath(misplacedProject2))
+        .andReturn(alternateBasePath).anyTimes();
+    expect(configMock.getAllBasePaths())
+        .andReturn(Arrays.asList(alternateBasePath)).anyTimes();
+    replay(configMock);
+
+    repoManager.createRepository(basePathProject);
+    repoManager.createRepository(altPathProject);
+    // create the misplaced ones without the repomanager otherwise they would
+    // end up at the proper place.
+    createRepository(repoManager.getBasePath(basePathProject),
+        misplacedProject2);
+    createRepository(alternateBasePath, misplacedProject1);
+
+    SortedSet<Project.NameKey> repoList = repoManager.list();
+    assertThat(repoList.size()).isEqualTo(2);
+    assertThat(repoList.toArray(new Project.NameKey[repoList.size()]))
+        .isEqualTo(new Project.NameKey[] {altPathProject, basePathProject});
+  }
+
+  private void createRepository(Path directory, Project.NameKey projectName)
+      throws IOException {
+    String n = projectName.get() + Constants.DOT_GIT_EXT;
+    FileKey loc = FileKey.exact(directory.resolve(n).toFile(), FS.DETECTED);
+    try (Repository db = RepositoryCache.open(loc, false)) {
+      db.create(true /* bare */);
+    }
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void testRelativeAlternateLocation() {
+    configMock = createNiceMock(RepositoryConfig.class);
+    expect(configMock.getAllBasePaths())
+        .andReturn(Arrays.asList(Paths.get("repos"))).anyTimes();
+    replay(configMock);
+    repoManager =
+        new MultiBaseLocalDiskRepositoryManager(site, cfg,
+            createNiceMock(NotesMigration.class), configMock);
+  }
+}

gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java

@@ -44,7 +44,7 @@ import com.google.gerrit.server.config.RestCacheAdminModule;
 import com.google.gerrit.server.config.SitePath;
 import com.google.gerrit.server.git.ChangeCacheImplModule;
 import com.google.gerrit.server.git.GarbageCollectionModule;
-import com.google.gerrit.server.git.LocalDiskRepositoryManager;
+import com.google.gerrit.server.git.GitRepositoryManagerModule;
 import com.google.gerrit.server.git.ReceiveCommitsExecutorModule;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.gerrit.server.index.IndexModule;
@@ -285,7 +285,6 @@ public class WebAppInitializer extends GuiceServletContextListener
       modules.add(new GerritServerConfigModule());
     }
     modules.add(new SchemaModule());
-    modules.add(new LocalDiskRepositoryManager.Module());
     modules.add(new ConfigNotesMigration.Module());
     modules.add(SchemaVersionCheck.module());
     modules.add(new AuthConfigModule());
@@ -296,6 +295,7 @@ public class WebAppInitializer extends GuiceServletContextListener
     final List<Module> modules = new ArrayList<>();
     modules.add(new DropWizardMetricMaker.RestModule());
     modules.add(new EventBroker.Module());
+    modules.add(cfgInjector.getInstance(GitRepositoryManagerModule.class));
     modules.add(new ChangeHookRunner.Module());
     modules.add(new ReceiveCommitsExecutorModule());
     modules.add(new DiffExecutorModule());