Add truncate after dedupe.
This commit is contained in:
parent
751e5fc15e
commit
021d5c743b
@ -8,6 +8,8 @@ import org.skife.jdbi.v2.Handle;
|
|||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.sql.SQLException;
|
||||||
|
|
||||||
public class MonDeDuper implements Managed {
|
public class MonDeDuper implements Managed {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(MonDeDuper.class);
|
private static Logger logger = LoggerFactory.getLogger(MonDeDuper.class);
|
||||||
@ -51,17 +53,23 @@ public class MonDeDuper implements Managed {
|
|||||||
private static final String DEDEUP_STAGING_DIMS =
|
private static final String DEDEUP_STAGING_DIMS =
|
||||||
"insert into MonMetrics.Dimensions select distinct * from MonMetrics.StagedDimensions where metric_definition_id not in (select metric_definition_id from MonMetrics.Dimensions)";
|
"insert into MonMetrics.Dimensions select distinct * from MonMetrics.StagedDimensions where metric_definition_id not in (select metric_definition_id from MonMetrics.Dimensions)";
|
||||||
|
|
||||||
private static final String DROP_PARTITION_STAGING_DEFS =
|
private static final String TRUNCATE_STAGING_DEFS =
|
||||||
"select drop partition('monmetrics.stageddefinitions', )";
|
"truncate table monmetrics.stageddefinitions";
|
||||||
|
|
||||||
private static final String DROP_PARTITION_STAGING_DIMS =
|
private static final String TRUNCATE_STAGING_DIMS =
|
||||||
"select drop partition('monmetrics.stageddimensions', )";
|
"truncate table monmetrics.stageddimensions";
|
||||||
|
|
||||||
private DeDuperRunnable(MonPersisterConfiguration configuration, DBI dbi) {
|
private DeDuperRunnable(MonPersisterConfiguration configuration, DBI dbi) {
|
||||||
this.configuration = configuration;
|
this.configuration = configuration;
|
||||||
this.dbi = dbi;
|
this.dbi = dbi;
|
||||||
this.handle = this.dbi.open();
|
this.handle = this.dbi.open();
|
||||||
this.handle.execute("SET TIME ZONE TO 'UTC'");
|
this.handle.execute("SET TIME ZONE TO 'UTC'");
|
||||||
|
try {
|
||||||
|
this.handle.getConnection().setAutoCommit(false);
|
||||||
|
} catch (SQLException e) {
|
||||||
|
logger.error("Failed to set autocommit to false", e);
|
||||||
|
System.exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -71,13 +79,25 @@ public class MonDeDuper implements Managed {
|
|||||||
for (; ; ) {
|
for (; ; ) {
|
||||||
try {
|
try {
|
||||||
Thread.sleep(seconds * 1000);
|
Thread.sleep(seconds * 1000);
|
||||||
|
handle.begin();
|
||||||
logger.debug("Waited " + seconds + " seconds");
|
logger.debug("Waited " + seconds + " seconds");
|
||||||
|
|
||||||
logger.debug("Executing:" + DEDUPE_STAGING_DEFS);
|
logger.debug("Executing: " + DEDUPE_STAGING_DEFS);
|
||||||
handle.execute(DEDUPE_STAGING_DEFS);
|
handle.execute(DEDUPE_STAGING_DEFS);
|
||||||
|
|
||||||
logger.debug("Executing:" + DEDEUP_STAGING_DIMS);
|
logger.debug("Executing: " + TRUNCATE_STAGING_DEFS);
|
||||||
|
handle.execute(TRUNCATE_STAGING_DEFS);
|
||||||
|
|
||||||
|
handle.commit();
|
||||||
|
|
||||||
|
handle.begin();
|
||||||
|
logger.debug("Executing: " + DEDEUP_STAGING_DIMS);
|
||||||
handle.execute(DEDEUP_STAGING_DIMS);
|
handle.execute(DEDEUP_STAGING_DIMS);
|
||||||
|
|
||||||
|
logger.debug("Executing: " + TRUNCATE_STAGING_DIMS);
|
||||||
|
handle.execute(TRUNCATE_STAGING_DIMS);
|
||||||
|
handle.commit();
|
||||||
|
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
logger.warn("Failed to wait for " + seconds + " between deduping", e);
|
logger.warn("Failed to wait for " + seconds + " between deduping", e);
|
||||||
}
|
}
|
||||||
|
@ -41,7 +41,8 @@ databaseConfiguration:
|
|||||||
# url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som
|
# url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som
|
||||||
url: jdbc:vertica://15.185.94.245:5433/som
|
url: jdbc:vertica://15.185.94.245:5433/som
|
||||||
# user: persister
|
# user: persister
|
||||||
user: mon_persister
|
# user: mon_persister
|
||||||
|
user: dbadmin
|
||||||
password: password
|
password: password
|
||||||
properties:
|
properties:
|
||||||
ssl: false
|
ssl: false
|
||||||
|
Loading…
Reference in New Issue
Block a user