Fix metric-list limits

Change-Id: I5ad17fa686cd00ecaff7ced34821e6cb05528a05
This commit is contained in:
Ryan Brandt 2016-04-19 12:10:41 -06:00
parent 5697d16f42
commit 88b756cf64
5 changed files with 155 additions and 145 deletions

View File

@ -29,7 +29,6 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Iterator;
import java.util.Set;
import javax.annotation.Nullable;
@ -53,6 +52,8 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo {
public static final DateTimeFormatter DATETIME_FORMATTER =
ISODateTimeFormat.dateTime().withZoneUTC();
public static final ByteBuffer EMPTY_DEF_ID = ByteBuffer.wrap(new byte[0]);
private static final String FIND_BY_METRIC_DEF_SQL =
"select mes.definition_dimensions_id, "
+ "mes.time_stamp, mes.value, mes.value_meta "
@ -66,14 +67,9 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo {
private static final String
DEFDIM_IDS_SELECT =
"SELECT defDims.id, defDims.dimension_set_id, defDims.definition_id "
+ "FROM MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defDims "
+ "WHERE defDims.definition_id = def.id "
+ "AND def.tenant_id = :tenantId "
+ "%s " // Name clause here
+ "%s;"; // Dimensions and clause goes here
private static final String TABLE_TO_JOIN_DIMENSIONS_ON = "defDims";
"SELECT defDims.id "
+ "FROM MonMetrics.DefinitionDimensions defDims "
+ "WHERE defDims.id IN (%s)";
private final DBI db;
@ -104,45 +100,29 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo {
Map<ByteBuffer, Measurements> results = new LinkedHashMap<>();
Set<byte[]> defDimIdSet = new HashSet<>();
Set<byte[]> dimSetIdSet = new HashSet<>();
String namePart = "";
if (name != null && !name.isEmpty()) {
namePart = "AND def.name = :name ";
}
String defDimSql = String.format(
DEFDIM_IDS_SELECT,
namePart,
MetricQueries.buildDimensionAndClause(dimensions, "defDims", 0));
MetricQueries.buildMetricDefinitionSubSql(name, dimensions));
Query<Map<String, Object>> query = h.createQuery(defDimSql).bind("tenantId", tenantId);
MetricQueries.bindDimensionsToQuery(query, dimensions);
if (name != null && !name.isEmpty()) {
query.bind("name", name);
}
List<Map<String, Object>> rows = query.list();
MetricQueries.bindDimensionsToQuery(query, dimensions);
ByteBuffer defId = ByteBuffer.wrap(new byte[0]);
List<Map<String, Object>> rows = query.list();
for (Map<String, Object> row : rows) {
byte[] defDimId = (byte[]) row.get("id");
defDimIdSet.add(defDimId);
byte[] dimSetIdBytes = (byte[]) row.get("dimension_set_id");
dimSetIdSet.add(dimSetIdBytes);
byte[] defIdBytes = (byte[]) row.get("definition_id");
defId = ByteBuffer.wrap(defIdBytes);
}
if (!Boolean.TRUE.equals(mergeMetricsFlag) && (dimSetIdSet.size() > 1)) {
if (!Boolean.TRUE.equals(mergeMetricsFlag) && (defDimIdSet.size() > 1)) {
throw new MultipleMetricsException(name, dimensions);
}
@ -211,7 +191,7 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo {
}
Measurements measurements = (Boolean.TRUE.equals(mergeMetricsFlag)) ? results.get(defId) : results.get(defdimsId);
Measurements measurements = (Boolean.TRUE.equals(mergeMetricsFlag)) ? results.get(EMPTY_DEF_ID) : results.get(defdimsId);
if (measurements == null) {
if (Boolean.TRUE.equals(mergeMetricsFlag)) {
@ -219,10 +199,10 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo {
new Measurements(name, new HashMap<String, String>(),
new ArrayList<Object[]>());
results.put(defId, measurements);
results.put(EMPTY_DEF_ID, measurements);
} else {
measurements =
new Measurements(name, MetricQueries.dimensionsFor(h, (byte[]) dimSetIdSet.toArray()[0]),
new Measurements(name, MetricQueries.dimensionsFor(h, (byte[]) defDimIdSet.toArray()[0]),
new ArrayList<Object[]>());
results.put(defdimsId, measurements);
}

View File

@ -44,40 +44,46 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
logger =
LoggerFactory.getLogger(MetricDefinitionVerticaRepoImpl.class);
private static final String
FIND_METRIC_DEFS_SQL =
private static final String FIND_METRIC_DEFS_SQL =
"SELECT defDims.id as defDimsId, def.name, dims.name as dName, dims.value AS dValue "
+ "FROM MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defDims "
+ "FROM MonMetrics.Definitions def "
+ "JOIN MonMetrics.DefinitionDimensions defDims ON def.id = defDims.definition_id "
// Outer join needed in case there are no dimensions for a definition.
+ "LEFT OUTER JOIN MonMetrics.Dimensions dims ON dims.dimension_set_id = defDims"
+ ".dimension_set_id WHERE def.id = defDims.definition_id "
+ "and def.tenant_id = :tenantId "
+ "%s " // Name goes here.
+ "%s " // Offset goes here.
+ "%s " // Dimensions and clause goes here
+ "%s " // Optional timestamp qualifier goes here
+ ".dimension_set_id "
+ "WHERE defDims.id in (%s) "
+ "ORDER BY defDims.id ASC";
private static final String
FIND_METRIC_NAMES_SQL =
private static final String METRIC_DEF_SUB_QUERY =
"SELECT defDimsSub.id "
+ "FROM MonMetrics.Definitions defSub "
+ "JOIN MonMetrics.DefinitionDimensions defDimsSub ON defSub.id = defDimsSub.definition_id "
+ "WHERE defSub.tenant_id = :tenantId "
+ "%s " // Name goes here
+ "%s " // Offset goes here
+ "%s " // Dimensions and clause goes here
+ "%s " // Time qualifier goes here
+ "GROUP BY defDimsSub.id "
+ "ORDER BY defDimsSub.id ASC "
+ "%s "; // limit goes here
private static final String FIND_METRIC_NAMES_SQL =
"SELECT distinct def.id, def.name "
+ "FROM MonMetrics.Definitions def "
+ "WHERE def.id IN (%s) " // Subselect goes here
+ "ORDER BY def.id ASC ";
private static final String
METRIC_NAMES_SUB_SELECT =
private static final String METRIC_NAMES_SUB_SELECT =
"SELECT distinct MAX(defSub.id) as max_id " // The aggregation function gives us one id per name
+ "FROM MonMetrics.Definitions defSub, MonMetrics.DefinitionDimensions defDimsSub "
+ "WHERE defDimsSub.definition_id = defSub.id "
+ "AND defSub.tenant_id = :tenantId "
+ "FROM MonMetrics.Definitions defSub "
+ "JOIN MonMetrics.DefinitionDimensions defDimsSub ON defDimsSub.definition_id = defSub.id "
+ "WHERE defSub.tenant_id = :tenantId "
+ "%s " // Offset goes here.
+ "%s " // Dimensions and clause goes here
+ "GROUP BY defSub.name " // This is to reduce the (id, name) sets to only include unique names
+ "ORDER BY max_id ASC %s"; // Limit goes here.
private static final String
DEFDIM_IDS_SELECT =
private static final String DEFDIM_IDS_SELECT =
"SELECT defDims.id "
+ "FROM MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defDims "
+ "WHERE defDims.definition_id = def.id "
@ -85,9 +91,8 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
+ "%s " // Name clause here
+ "%s;"; // Dimensions and clause goes here
private static final String
MEASUREMENT_AND_CLAUSE =
"AND defDims.id IN ("
private static final String MEASUREMENT_AND_CLAUSE =
"AND defDimsSub.id IN ("
+ "SELECT definition_dimensions_id FROM MonMetrics.Measurements "
+ "WHERE to_hex(definition_dimensions_id) "
+ "%s " // List of definition dimension ids here
@ -152,8 +157,7 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
String.format(METRIC_NAMES_SUB_SELECT,
offsetPart,
MetricQueries.buildDimensionAndClause(dimensions,
TABLE_TO_JOIN_DIMENSIONS_ON,
0), // No limit on dim ids
TABLE_TO_JOIN_DIMENSIONS_ON),
limitPart);
String sql = String.format(FIND_METRIC_NAMES_SQL, defSubSelect);
@ -254,7 +258,7 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
if (name != null && !name.isEmpty()) {
namePart = " and def.name = :name ";
namePart = " and defSub.name = :name ";
}
@ -262,7 +266,15 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
if (offset != null && !offset.isEmpty()) {
offsetPart = " and defDims.id > :offset ";
offsetPart = " and defDimsSub.id > :offset ";
}
String limitPart = "";
if (limit > 0) {
limitPart = "limit " + Integer.toString(limit + 1);
}
@ -273,9 +285,14 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
String sql =
String.format(FIND_METRIC_DEFS_SQL,
namePart, offsetPart,
MetricQueries.buildDimensionAndClause(dimensions, "defDims", limit),
timeInClause);
String.format(METRIC_DEF_SUB_QUERY,
namePart,
offsetPart,
MetricQueries.buildDimensionAndClause(dimensions,
TABLE_TO_JOIN_DIMENSIONS_ON),
timeInClause,
limitPart)
);
Query<Map<String, Object>> query = h.createQuery(sql).bind("tenantId", tenantId);
@ -338,7 +355,7 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
String defDimSql = String.format(
DEFDIM_IDS_SELECT,
namePart,
MetricQueries.buildDimensionAndClause(dimensions, "defDims", 0));
MetricQueries.buildDimensionAndClause(dimensions, "defDims"));
Query<Map<String, Object>> query = dbHandle.createQuery(defDimSql).bind("tenantId", tenantId);

View File

@ -32,61 +32,84 @@ import monasca.common.persistence.SqlQueries;
* Vertica utilities for building metric queries.
*/
final class MetricQueries {
private static Splitter BAR_SPLITTER = Splitter.on('|').omitEmptyStrings().trimResults();
static final String METRIC_DEF_SUB_SQL =
"SELECT defDimsSub.id "
+ "FROM MonMetrics.Definitions as defSub "
+ "JOIN MonMetrics.DefinitionDimensions as defDimsSub ON defDimsSub.definition_id = defSub.id "
+ "WHERE defSub.tenant_id = :tenantId "
+ "%s " // metric name here
+ "%s " // dimension and clause here
+ "GROUP BY defDimsSub.id";
private static final String TABLE_TO_JOIN_DIMENSIONS_ON = "defDimsSub";
private MetricQueries() {}
static String buildDimensionAndClause(Map<String, String> dimensions,
String tableToJoinName,
int limit) {
static String buildMetricDefinitionSubSql(String name, Map<String, String> dimensions) {
StringBuilder sb = null;
String namePart = "";
if (dimensions != null && dimensions.size() > 0) {
int numDims = dimensions.size();
sb = new StringBuilder();
sb.append(" and " + tableToJoinName + ".dimension_set_id in ")
.append("(select dimension_set_id from MonMetrics.Dimensions where ");
int i = 0;
for (Iterator<Map.Entry<String, String>> it = dimensions.entrySet().iterator(); it.hasNext(); i++) {
Map.Entry<String, String> entry = it.next();
sb.append("name = :dname").append(i);
String dim_value = entry.getValue();
if (!Strings.isNullOrEmpty(dim_value)) {
List<String> values = Splitter.on('|').splitToList(dim_value);
if (values.size() > 1) {
sb.append(" and ( ");
for (int j = 0; j < values.size(); j++) {
sb.append("value = :dvalue").append(i).append('_').append(j);
if (j < values.size() - 1) {
sb.append(" or ");
}
}
sb.append(" )");
} else {
sb.append(" and value = :dvalue").append(i);
}
}
if (it.hasNext()) {
sb.append(" or ");
}
}
sb.append(" group by dimension_set_id")
.append(" having count(*) = " + numDims +" ");
//
// Limit is non-deterministic here unless we also
// order by.
//
if (limit > 0) {
sb.append("order by dimension_set_id ")
.append("limit " + Integer.toString(limit + 1));
}
sb.append(")");
if (name != null && !name.isEmpty()) {
namePart = "AND defSub.name = :name ";
}
return sb == null ? "" : sb.toString();
return String.format(METRIC_DEF_SUB_SQL,
namePart,
buildDimensionAndClause(dimensions,
TABLE_TO_JOIN_DIMENSIONS_ON));
}
static String buildDimensionAndClause(Map<String, String> dimensions,
String tableToJoinName) {
if (dimensions == null || dimensions.isEmpty()) {
return "";
}
StringBuilder sb = new StringBuilder();
sb.append(" and ").append(tableToJoinName).append(
".dimension_set_id in ( "
+ "SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE (");
int i = 0;
for (Iterator<Map.Entry<String, String>> it = dimensions.entrySet().iterator(); it.hasNext(); i++) {
Map.Entry<String, String> entry = it.next();
sb.append("(name = :dname").append(i);
String dim_value = entry.getValue();
if (!Strings.isNullOrEmpty(dim_value)) {
List<String> values = BAR_SPLITTER.splitToList(dim_value);
if (values.size() > 1) {
sb.append(" and ( ");
for (int j = 0; j < values.size(); j++) {
sb.append("value = :dvalue").append(i).append('_').append(j);
if (j < values.size() - 1) {
sb.append(" or ");
}
}
sb.append(")");
} else {
sb.append(" and value = :dvalue").append(i);
}
}
sb.append(")");
if (it.hasNext()) {
sb.append(" or ");
}
}
sb.append(") GROUP BY dimension_set_id HAVING count(*) = ").append(dimensions.size()).append(") ");
return sb.toString();
}
static void bindDimensionsToQuery(Query<?> query, Map<String, String> dimensions) {
@ -113,8 +136,11 @@ final class MetricQueries {
static Map<String, String> dimensionsFor(Handle handle, byte[] dimensionSetId) {
return SqlQueries.keyValuesFor(handle, "select name, value from MonMetrics.Dimensions "
+ "where" + " dimension_set_id = ?", dimensionSetId);
return SqlQueries.keyValuesFor(handle,
"select name, value from MonMetrics.Dimensions as d "
+ "join MonMetrics.DefinitionDimensions as dd "
+ "on d.dimension_set_id = dd.dimension_set_id "
+ "where" + " dd.id = ?", dimensionSetId);
}
static String createDefDimIdInClause(Set<byte[]> defDimIdSet) {

View File

@ -50,13 +50,10 @@ public class StatisticVerticaRepoImpl implements StatisticRepo {
"select defdims.id, def.name, d.name as dname, d.value as dvalue "
+ "from MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defdims "
+ "left outer join MonMetrics.Dimensions d on d.dimension_set_id = defdims.dimension_set_id "
+ "where def.id = defdims.definition_id and def.tenant_id = :tenantId "
+ "%s " // metric name here
+ "%s " // dimension and clause here
+ "where def.id = defdims.definition_id "
+ "and defdims.id in (%s) "
+ "order by defdims.id ASC";
private static final String TABLE_TO_JOIN_DIMENSIONS_ON = "defdims";
private final DBI db;
@Inject
@ -195,19 +192,9 @@ public class StatisticVerticaRepoImpl implements StatisticRepo {
List<byte[]> bytes = new ArrayList<>();
StringBuilder sb = new StringBuilder();
if (name != null && !name.isEmpty()) {
sb.append(" and def.name = :name");
}
String sql =
String
.format(FIND_BY_METRIC_DEF_SQL,
sb,
MetricQueries.buildDimensionAndClause(dimensions, TABLE_TO_JOIN_DIMENSIONS_ON, 0));
String sql = String.format(
FIND_BY_METRIC_DEF_SQL,
MetricQueries.buildMetricDefinitionSubSql(name, dimensions));
Query<Map<String, Object>> query =
h.createQuery(sql)

View File

@ -28,55 +28,55 @@ public class MetricQueriesTest {
public void metricQueriesBuildDimensionAndClauseTest1() {
String expectedResult =
" and defdims.dimension_set_id in (select dimension_set_id from MonMetrics.Dimensions "
+ "where name = :dname0 and value = :dvalue0 or name = :dname1 and value = :dvalue1 "
+ "group by dimension_set_id having count(*) = 2 order by dimension_set_id limit 2)";
" and defdims.dimension_set_id in ( SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE"
+ " ((name = :dname0 and value = :dvalue0) or (name = :dname1 and value = :dvalue1))"
+ " GROUP BY dimension_set_id HAVING count(*) = 2) ";
Map<String, String> dimsMap = new HashMap<>();
dimsMap.put("foo", "bar");
dimsMap.put("biz", "baz");
String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 1);
String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON);
assertEquals(expectedResult, s);
}
public void metricQueriesBuildDimensionAndClauseTest2() {
String expectedResult = "";
Map<String, String> dimsMap = new HashMap<>();
assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap,TABLE_TO_JOIN_DIMENSIONS_ON, 0));
assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON));
}
public void metricQueriesBuildDimensionAndClauseForTest3() {
String expectedResult = "";
Map<String, String> dimsMap = null;
assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 0));
assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON));
}
public void metricQueriesBuildDimensionAndClauseTest4() {
String expectedResult =
" and defdims.dimension_set_id in (select dimension_set_id from MonMetrics.Dimensions "
+ "where name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1 ) "
+ "group by dimension_set_id having count(*) = 1 order by dimension_set_id limit 2)";
" and defdims.dimension_set_id in ( SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE"
+ " ((name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1)))"
+ " GROUP BY dimension_set_id HAVING count(*) = 1) ";
Map<String, String> dimsMap = new HashMap<>();
dimsMap.put("foo", "bar|baz");
String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 1);
String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON);
assertEquals(expectedResult, s);
}
public void metricQueriesBuildDimensionAndClauseTest5() {
String expectedResult =
" and defdims.dimension_set_id in (select dimension_set_id from MonMetrics.Dimensions "
+ "where name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1 ) "
+ "or name = :dname1 and ( value = :dvalue1_0 or value = :dvalue1_1 ) "
+ "group by dimension_set_id having count(*) = 2 order by dimension_set_id limit 2)";
" and defdims.dimension_set_id in ( SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE"
+ " ((name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1))"
+ " or (name = :dname1 and ( value = :dvalue1_0 or value = :dvalue1_1)))"
+ " GROUP BY dimension_set_id HAVING count(*) = 2) ";
Map<String, String> dimsMap = new HashMap<>();
dimsMap.put("foo", "bar|baz");
dimsMap.put("biz", "baz|baf");
String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 1);
String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON);
assertEquals(expectedResult, s);
}
}