opendev
/
gerrit
Code Issues Proposed changes

Limit threads used in Reindex to DB connection pool size 

We open one database connection per thread, so it is not possible to
run more than database.poolSize threads total if pooling is enabled.
Limit the number of threads used by Reindex to this total.

Change-Id: Iac711896f845fa17f967ec3a322f253a3cb6fc59
This commit is contained in:
Dave Borowitz 2013-10-02 16:13:16 -07:00
parent d4eee8bc99
commit 7e057414e2
2 changed files with 26 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
View File

@ -29,6 +29,7 @@ import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.cache.CacheRemovalListener;
import com.google.gerrit.server.cache.h2.DefaultCacheFactory;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.ChangeBatchIndexer;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeSchemas;
@ -37,6 +38,8 @@ import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.NoIndexModule;
import com.google.gerrit.server.patch.PatchListCacheImpl;
import com.google.gerrit.server.schema.DataSourceProvider;
import com.google.gerrit.server.schema.DataSourceType;
import com.google.gerrit.solr.SolrIndexModule;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
@ -48,10 +51,13 @@ import com.google.inject.Provider;
import com.google.inject.ProvisionException;
import com.google.inject.TypeLiteral;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.TextProgressMonitor;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.kohsuke.args4j.Option;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
@ -59,6 +65,8 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
public class Reindex extends SiteProgram {
private static final Logger log = LoggerFactory.getLogger(Reindex.class);
@Option(name = "--threads", usage = "Number of threads to use for indexing")
private int threads = Runtime.getRuntime().availableProcessors();
@ -86,6 +94,7 @@ public class Reindex extends SiteProgram {
if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) {
throw die("index.type must be configured (or not SQL)");
}
limitThreads();
if (version == null) {
version = ChangeSchemas.getLatest().getVersion();
}
@ -109,6 +118,20 @@ public class Reindex extends SiteProgram {
return result;
}
private void limitThreads() {
Config cfg =
dbInjector.getInstance(Key.get(Config.class, GerritServerConfig.class));
boolean usePool = cfg.getBoolean("database", "connectionpool",
dbInjector.getInstance(DataSourceType.class).usePool());
int poolLimit = cfg.getInt("database", "poollimit",
DataSourceProvider.DEFAULT_POOL_LIMIT);
if (usePool && threads > poolLimit) {
log.warn("Limiting reindexing to " + poolLimit
+ " threads due to database.poolLimit");
threads = poolLimit;
}
}
private Injector createSysInjector() {
List<Module> modules = Lists.newArrayList();
modules.add(PatchListCacheImpl.module());

4
gerrit-server/src/main/java/com/google/gerrit/server/schema/DataSourceProvider.java
View File

@ -41,6 +41,8 @@ import javax.sql.DataSource;
@Singleton
public class DataSourceProvider implements Provider<DataSource>,
LifecycleListener {
public static final int DEFAULT_POOL_LIMIT = 8;
private final SitePaths site;
private final Config cfg;
private final Context ctx;
@ -118,7 +120,7 @@ public class DataSourceProvider implements Provider<DataSource>,
if (password != null && !password.isEmpty()) {
ds.setPassword(password);
}
ds.setMaxActive(cfg.getInt("database", "poollimit", 8));
ds.setMaxActive(cfg.getInt("database", "poollimit", DEFAULT_POOL_LIMIT));
ds.setMinIdle(cfg.getInt("database", "poolminidle", 4));
ds.setMaxIdle(cfg.getInt("database", "poolmaxidle", 4));
ds.setMaxWait(ConfigUtil.getTimeUnit(cfg, "database", null,