From 0958da70574f0b70e2ac19a12480a1124ad3bb70 Mon Sep 17 00:00:00 2001 From: Deklan Dieterly Date: Thu, 10 Apr 2014 14:50:31 -0600 Subject: [PATCH 1/2] Migrate to Dropwizard 0.7.0. --- pom.xml | 23 ++-- ...vice.java => MonPersisterApplication.java} | 29 +++-- .../mon/persister/MonPersisterModule.java | 2 +- .../MonPersisterConfiguration.java | 10 +- .../AlarmStateTransitionsConsumer.java | 2 +- .../persister/consumer/MetricsConsumer.java | 2 +- .../mon/persister/dbi/DBIProvider.java | 6 +- .../event/MetricMessageEventHandler.java | 37 +++--- .../healthcheck/SimpleHealthCheck.java | 6 +- .../repository/RepositoryCommitHeartbeat.java | 2 +- .../repository/VerticaMetricRepository.java | 21 ++-- src/main/resources/mon-persister-config.yml | 106 +++++------------- 12 files changed, 107 insertions(+), 139 deletions(-) rename src/main/java/com/hpcloud/mon/persister/{MonPersisterService.java => MonPersisterApplication.java} (67%) diff --git a/pom.xml b/pom.xml index 923cb2c7..5ea1c10f 100644 --- a/pom.xml +++ b/pom.xml @@ -64,9 +64,14 @@ - com.yammer.dropwizard + io.dropwizard dropwizard-core - 0.6.2 + 0.7.0 + + + io.dropwizard + dropwizard-jdbi + 0.7.0 com.google.inject @@ -93,11 +98,7 @@ disruptor 3.2.0 - - com.yammer.dropwizard - dropwizard-jdbi - 0.6.2 - + com.vertica vertica-jdbc @@ -118,11 +119,6 @@ commons-codec 1.5 - - com.yammer.metrics - metrics-core - 2.2.0 - @@ -169,7 +165,8 @@ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - com.hpcloud.mon.persister.MonPersisterService + com.hpcloud.mon.persister.MonPersisterApplication + diff --git a/src/main/java/com/hpcloud/mon/persister/MonPersisterService.java b/src/main/java/com/hpcloud/mon/persister/MonPersisterApplication.java similarity index 67% rename from src/main/java/com/hpcloud/mon/persister/MonPersisterService.java rename to src/main/java/com/hpcloud/mon/persister/MonPersisterApplication.java index 5e8beb8a..7b744522 100644 --- a/src/main/java/com/hpcloud/mon/persister/MonPersisterService.java +++ b/src/main/java/com/hpcloud/mon/persister/MonPersisterApplication.java @@ -5,24 +5,28 @@ import com.google.inject.Injector; import com.hpcloud.mon.persister.configuration.MonPersisterConfiguration; import com.hpcloud.mon.persister.consumer.AlarmStateTransitionsConsumer; import com.hpcloud.mon.persister.consumer.MetricsConsumer; -import com.hpcloud.mon.persister.repository.RepositoryCommitHeartbeat; import com.hpcloud.mon.persister.healthcheck.SimpleHealthCheck; +import com.hpcloud.mon.persister.repository.RepositoryCommitHeartbeat; import com.hpcloud.mon.persister.resource.Resource; -import com.yammer.dropwizard.Service; -import com.yammer.dropwizard.config.Bootstrap; -import com.yammer.dropwizard.config.Environment; +import io.dropwizard.Application; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; import javax.inject.Inject; -public class MonPersisterService extends Service { +public class MonPersisterApplication extends Application { public static void main(String[] args) throws Exception { - new MonPersisterService().run(args); + new MonPersisterApplication().run(args); } @Override public void initialize(Bootstrap bootstrap) { - bootstrap.setName("mon-persister"); + } + + @Override + public String getName() { + return "mon-persister"; } @Inject private kafka.javaapi.consumer.ConsumerConnector consumerConnector; @@ -33,18 +37,19 @@ public class MonPersisterService extends Service { Injector injector = Guice.createInjector(new MonPersisterModule(configuration, environment)); // Sample resource. - environment.addResource(new Resource()); + environment.jersey().register(new Resource()); // Sample health check. - environment.addHealthCheck(new SimpleHealthCheck("test-health-check")); + environment.healthChecks().register("test-health-check", new SimpleHealthCheck()); MetricsConsumer metricsConsumer = injector.getInstance(MetricsConsumer.class); - environment.manage(metricsConsumer); + environment.lifecycle().manage(metricsConsumer); AlarmStateTransitionsConsumer alarmStateTransitionsConsumer = injector.getInstance(AlarmStateTransitionsConsumer.class); - environment.manage(alarmStateTransitionsConsumer); + environment.lifecycle().manage(alarmStateTransitionsConsumer); RepositoryCommitHeartbeat repositoryCommitHeartbeat = injector.getInstance(RepositoryCommitHeartbeat.class); - environment.manage(repositoryCommitHeartbeat); + environment.lifecycle().manage(repositoryCommitHeartbeat); + } } \ No newline at end of file diff --git a/src/main/java/com/hpcloud/mon/persister/MonPersisterModule.java b/src/main/java/com/hpcloud/mon/persister/MonPersisterModule.java index a6235a9b..5a083811 100644 --- a/src/main/java/com/hpcloud/mon/persister/MonPersisterModule.java +++ b/src/main/java/com/hpcloud/mon/persister/MonPersisterModule.java @@ -13,7 +13,7 @@ import com.hpcloud.mon.persister.disruptor.event.MetricMessageEventHandler; import com.hpcloud.mon.persister.disruptor.event.MetricMessageEventHandlerFactory; import com.hpcloud.mon.persister.repository.RepositoryCommitHeartbeat; import com.lmax.disruptor.ExceptionHandler; -import com.yammer.dropwizard.config.Environment; +import io.dropwizard.setup.Environment; import org.skife.jdbi.v2.DBI; public class MonPersisterModule extends AbstractModule { diff --git a/src/main/java/com/hpcloud/mon/persister/configuration/MonPersisterConfiguration.java b/src/main/java/com/hpcloud/mon/persister/configuration/MonPersisterConfiguration.java index 3352129f..43458691 100644 --- a/src/main/java/com/hpcloud/mon/persister/configuration/MonPersisterConfiguration.java +++ b/src/main/java/com/hpcloud/mon/persister/configuration/MonPersisterConfiguration.java @@ -1,8 +1,8 @@ package com.hpcloud.mon.persister.configuration; import com.fasterxml.jackson.annotation.JsonProperty; -import com.yammer.dropwizard.config.Configuration; -import com.yammer.dropwizard.db.DatabaseConfiguration; +import io.dropwizard.Configuration; +import io.dropwizard.db.DataSourceFactory; import javax.validation.Valid; import javax.validation.constraints.NotNull; @@ -46,10 +46,10 @@ public class MonPersisterConfiguration extends Configuration { @Valid @NotNull @JsonProperty - private final DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(); + private final DataSourceFactory dataSourceFactory = new DataSourceFactory(); - public DatabaseConfiguration getDatabaseConfiguration() { - return databaseConfiguration; + public DataSourceFactory getDataSourceFactory() { + return dataSourceFactory; } @Valid diff --git a/src/main/java/com/hpcloud/mon/persister/consumer/AlarmStateTransitionsConsumer.java b/src/main/java/com/hpcloud/mon/persister/consumer/AlarmStateTransitionsConsumer.java index 5d864f98..9100902d 100644 --- a/src/main/java/com/hpcloud/mon/persister/consumer/AlarmStateTransitionsConsumer.java +++ b/src/main/java/com/hpcloud/mon/persister/consumer/AlarmStateTransitionsConsumer.java @@ -2,7 +2,7 @@ package com.hpcloud.mon.persister.consumer; import com.google.inject.Inject; import com.hpcloud.mon.persister.disruptor.AlarmStateHistoryDisruptor; -import com.yammer.dropwizard.lifecycle.Managed; +import io.dropwizard.lifecycle.Managed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/hpcloud/mon/persister/consumer/MetricsConsumer.java b/src/main/java/com/hpcloud/mon/persister/consumer/MetricsConsumer.java index 3edb14c9..805c655c 100644 --- a/src/main/java/com/hpcloud/mon/persister/consumer/MetricsConsumer.java +++ b/src/main/java/com/hpcloud/mon/persister/consumer/MetricsConsumer.java @@ -2,7 +2,7 @@ package com.hpcloud.mon.persister.consumer; import com.google.inject.Inject; import com.hpcloud.mon.persister.disruptor.MetricDisruptor; -import com.yammer.dropwizard.lifecycle.Managed; +import io.dropwizard.lifecycle.Managed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/hpcloud/mon/persister/dbi/DBIProvider.java b/src/main/java/com/hpcloud/mon/persister/dbi/DBIProvider.java index fd3a5913..63cbedee 100644 --- a/src/main/java/com/hpcloud/mon/persister/dbi/DBIProvider.java +++ b/src/main/java/com/hpcloud/mon/persister/dbi/DBIProvider.java @@ -2,8 +2,8 @@ package com.hpcloud.mon.persister.dbi; import com.google.inject.ProvisionException; import com.hpcloud.mon.persister.configuration.MonPersisterConfiguration; -import com.yammer.dropwizard.config.Environment; -import com.yammer.dropwizard.jdbi.DBIFactory; +import io.dropwizard.jdbi.DBIFactory; +import io.dropwizard.setup.Environment; import org.skife.jdbi.v2.DBI; import javax.inject.Inject; @@ -23,7 +23,7 @@ public class DBIProvider implements Provider { @Override public DBI get() { try { - return new DBIFactory().build(environment, configuration.getDatabaseConfiguration(), "vertica"); + return new DBIFactory().build(environment, configuration.getDataSourceFactory(), "vertica"); } catch (ClassNotFoundException e) { throw new ProvisionException("Failed to provision DBI", e); } diff --git a/src/main/java/com/hpcloud/mon/persister/disruptor/event/MetricMessageEventHandler.java b/src/main/java/com/hpcloud/mon/persister/disruptor/event/MetricMessageEventHandler.java index bd5d0034..6a184af5 100644 --- a/src/main/java/com/hpcloud/mon/persister/disruptor/event/MetricMessageEventHandler.java +++ b/src/main/java/com/hpcloud/mon/persister/disruptor/event/MetricMessageEventHandler.java @@ -1,5 +1,8 @@ package com.hpcloud.mon.persister.disruptor.event; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Meter; +import com.codahale.metrics.Timer; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.hpcloud.mon.persister.configuration.MonPersisterConfiguration; @@ -7,11 +10,7 @@ import com.hpcloud.mon.persister.message.MetricMessage; import com.hpcloud.mon.persister.repository.Sha1HashId; import com.hpcloud.mon.persister.repository.VerticaMetricRepository; import com.lmax.disruptor.EventHandler; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Meter; -import com.yammer.metrics.core.Timer; -import com.yammer.metrics.core.TimerContext; +import io.dropwizard.setup.Environment; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +20,6 @@ import java.util.Date; import java.util.Map; import java.util.TimeZone; import java.util.TreeMap; -import java.util.concurrent.TimeUnit; public class MetricMessageEventHandler implements EventHandler { @@ -40,24 +38,35 @@ public class MetricMessageEventHandler implements EventHandler definitionsIdCache; private final Cache dimensionsIdCache; @@ -66,15 +66,20 @@ public class VerticaMetricRepository extends VerticaRepository { private final String dimensionsTempStagingTableInsertStmt; private final String definitionDimensionsTempStagingTableInsertStmt; - private final Timer commitTimer = Metrics.newTimer(this.getClass(), "commits-timer"); - private final Timer flushTimer = Metrics.newTimer(this.getClass(), "staging-tables-flushed-timer"); + private final Timer commitTimer; + private final Timer flushTimer; @Inject - public VerticaMetricRepository(DBI dbi, MonPersisterConfiguration configuration) throws NoSuchAlgorithmException, SQLException { + public VerticaMetricRepository(DBI dbi, MonPersisterConfiguration configuration, + Environment environment) throws NoSuchAlgorithmException, SQLException { super(dbi); logger.debug("Instantiating: " + this); this.configuration = configuration; + this.environment = environment; + this.commitTimer = this.environment.metrics().timer(this.getClass().getName() + "." + "commits-timer"); + this.flushTimer = this.environment.metrics().timer(this.getClass().getName() + "." + "staging-tables-flushed-timer"); + definitionsIdCache = CacheBuilder.newBuilder() .maximumSize(configuration.getVerticaMetricRepositoryConfiguration().getMaxCacheSize()).build(); @@ -149,7 +154,7 @@ public class VerticaMetricRepository extends VerticaRepository { public void flush() { commitBatch(); long startTime = System.currentTimeMillis(); - TimerContext context = flushTimer.time(); + Timer.Context context = flushTimer.time(); handle.execute(definitionsTempStagingTableInsertStmt); handle.execute("truncate table " + definitionsTempStagingTableName); handle.execute(dimensionsTempStagingTableInsertStmt); @@ -166,7 +171,7 @@ public class VerticaMetricRepository extends VerticaRepository { private void commitBatch() { long startTime = System.currentTimeMillis(); - TimerContext context = commitTimer.time(); + Timer.Context context = commitTimer.time(); metricsBatch.execute(); stagedDefinitionsBatch.execute(); stagedDimensionsBatch.execute(); diff --git a/src/main/resources/mon-persister-config.yml b/src/main/resources/mon-persister-config.yml index fdb3e6b7..7f898c46 100644 --- a/src/main/resources/mon-persister-config.yml +++ b/src/main/resources/mon-persister-config.yml @@ -3,7 +3,7 @@ name: mon-persister #Kafka settings. kafkaConfiguration: # See http://kafka.apache.org/documentation.html#api for semantics and defaults. - topic: test + topic: metrics numThreads: 1 groupId: 1 zookeeperConnect: localhost:2181 @@ -39,7 +39,7 @@ monDeDuperConfiguration: verticaMetricRepositoryConfiguration: maxCacheSize: 2000000 -databaseConfiguration: +dataSourceFactory: driverClass: com.vertica.jdbc.Driver # url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som url: jdbc:vertica://15.185.94.245:5433/som @@ -59,18 +59,16 @@ databaseConfiguration: minSize: 8 # the maximum number of connections to keep open - - maxSize: 41 # whether or not idle connections should be validated checkConnectionWhileIdle: false - # how long a connection must be held before it can be validated - checkConnectionHealthWhenIdleFor: 10s - # the maximum lifetime of an idle connection - closeConnectionIfIdleFor: 1 minute + maxConnectionAge: 1 minute + +metrics: + frequency: 1 second # Logging settings. logging: @@ -84,80 +82,34 @@ logging: # Sets the level for 'com.example.app' to DEBUG. com.example.app: DEBUG -# com.hpcloud: debug + com.hpcloud: debug # com.hpcloud.mon.persister.repository: DEBUG - # Settings for logging to stdout. - console: + appenders: + - type: console + threshold: INFO + timeZone: UTC + target: stdout - # If true, write log statements to stdout. - enabled: true + - type: file + threshold: INFO + archive: true + currentLogFilename: ./logs/mon-persister.log + archivedLogFilenamePattern: ./logs/mon-persister-%d.log.gz + archivedFileCount: 5 - # Do not display log statements below this threshold to stdout. - threshold: ALL + - type: file + # The file to which current statements will be logged. + currentLogFilename: ./logs/mon-persister.log - # The time zone in which dates should be displayed. - timeZone: UTC + # When the log file rotates, the archived log will be renamed to this and gzipped. The + # %d is replaced with the previous day (yyyy-MM-dd). Custom rolling windows can be created + # by passing a SimpleDateFormat-compatible format as an argument: "%d{yyyy-MM-dd-hh}". + archivedLogFilenamePattern: ./logs/mon-persister-%d.log.gz - # A custom Logback format string. - logFormat: null + # The number of archived files to keep. + archivedFileCount: 5 - # Settings for logging to a file. - file: + # The timezone used to format dates. HINT: USE THE DEFAULT, UTC. + timeZone: UTC - # If true, write log statements to a file. - enabled: true - - # Do not write log statements below this threshold to the file. - threshold: ALL - - # The time zone in which dates should be displayed. - timeZone: UTC - - # A custom Logback format string. - logFormat: null - - # The file to which statements will be logged. - # - # If enabled is true, this must be specified. - currentLogFilename: ./logs/mon-persister.log - - # If true, log files are rotated and archived. - archive: true - - # When the log file rolls over, the file will be archived to - # app-2012-03-15.log.gz, example.log will be truncated, - # and new statements written to it. - # - # If archive is true, this must be specified. - archivedLogFilenamePattern: ./logs/mon-persister-%d.log.gz - - # The maximum number of log files to archive. - archivedFileCount: 5 - - # Settings for logging to syslog. - syslog: - - # If true, write log statements to syslog. - enabled: false - - # The hostname of the syslog server to which statements will be - # sent. - # - # N.B.: If this is the local host, the local syslog instance - # will need to be configured to listen on an inet socket, not just - # a Unix socket. - host: localhost - - # The syslog facility to which statements will be sent. - # - # Can be one of: {AUTH, AUTHPRIV, DAEMON, CRON, FTP, LPR, KERN, - # MAIL, NEWS, SYSLOG, USER, UUCP, LOCAL0, LOCAL1, LOCAL2, LOCAL3, - # LOCAL4, LOCAL5, LOCAL6, LOCAL7}. - facility: local0 - - # The time zone in which dates should be displayed. - timeZone: UTC - - # A custom Logback format string. - logFormat: null \ No newline at end of file From ce82c0b8d625b5f304169b4581cb5748e2b57e70 Mon Sep 17 00:00:00 2001 From: Deklan Dieterly Date: Thu, 10 Apr 2014 15:30:53 -0600 Subject: [PATCH 2/2] Migrate to Dropwizard 0.7.0. Change main class to MonPersisterApplication. --- src/deb/init/mon-persister.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deb/init/mon-persister.conf b/src/deb/init/mon-persister.conf index 90829987..4bfdf4a2 100644 --- a/src/deb/init/mon-persister.conf +++ b/src/deb/init/mon-persister.conf @@ -8,4 +8,4 @@ respawn setgid persister setuid persister -exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.mon.persister.MonPersisterService server /etc/mon/persister-config.yml +exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.mon.persister.MonPersisterApplication server /etc/mon/persister-config.yml