Added support for changed sub alarm expressions which are sent in the AlarmUpdatedEvent

This commit is contained in:
Jonathan Halterman 2014-04-17 16:19:40 -07:00
parent a64f03738b
commit 410ea5e43c
5 changed files with 132 additions and 64 deletions

View File

@ -1,11 +1,10 @@
package com.hpcloud.mon.app;
import java.util.AbstractMap.SimpleEntry;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
@ -18,6 +17,8 @@ import kafka.producer.KeyedMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.Sets;
import com.hpcloud.mon.MonApiConfiguration;
import com.hpcloud.mon.app.command.UpdateAlarmCommand;
@ -59,6 +60,15 @@ public class AlarmService {
this.notificationMethodRepo = notificationMethodRepo;
}
static class SubExpressions {
/** Sub expressions which have been removed from an updated alarm expression. */
Map<String, AlarmSubExpression> oldAlarmSubExpressions;
/** Sub expressions which have had their operator or threshold changed. */
Map<String, AlarmSubExpression> changedSubExpressions;
/** Sub expressions which have been added to an updated alarm expression. */
Map<String, AlarmSubExpression> newAlarmSubExpressions;
}
/**
* Creates an alarm and publishes an AlarmCreatedEvent. Note, the event is published first since
* chances of failure are higher.
@ -132,10 +142,11 @@ public class AlarmService {
assertAlarmExists(tenantId, alarmId, command.alarmActions, command.okActions,
command.undeterminedActions);
updateInternal(tenantId, alarmId, false, command.name, command.description, command.expression,
alarmExpression, command.state, command.actionsEnabled, command.alarmActions, command.okActions,
command.undeterminedActions);
alarmExpression, command.state, command.actionsEnabled, command.alarmActions,
command.okActions, command.undeterminedActions);
return new Alarm(alarmId, command.name, command.description, command.expression, command.state,
command.actionsEnabled, command.alarmActions, command.okActions, command.undeterminedActions);
command.actionsEnabled, command.alarmActions, command.okActions,
command.undeterminedActions);
}
/**
@ -146,7 +157,7 @@ public class AlarmService {
* @throws InvalidEntityException if one of the actions cannot be found
*/
public Alarm patch(String tenantId, String alarmId, String name, String description,
String expression, AlarmExpression alarmExpression, AlarmState state, Boolean actionsEnabled,
String expression, AlarmExpression alarmExpression, AlarmState state, Boolean enabled,
List<String> alarmActions, List<String> okActions, List<String> undeterminedActions) {
Alarm alarm = assertAlarmExists(tenantId, alarmId, alarmActions, okActions, undeterminedActions);
name = name == null ? alarm.getName() : name;
@ -154,12 +165,12 @@ public class AlarmService {
expression = expression == null ? alarm.getExpression() : expression;
alarmExpression = alarmExpression == null ? AlarmExpression.of(expression) : alarmExpression;
state = state == null ? alarm.getState() : state;
actionsEnabled = actionsEnabled == null ? alarm.isActionsEnabled() : actionsEnabled;
enabled = enabled == null ? alarm.isActionsEnabled() : enabled;
updateInternal(tenantId, alarmId, true, name, description, expression, alarmExpression, state,
actionsEnabled, alarmActions, okActions, undeterminedActions);
enabled, alarmActions, okActions, undeterminedActions);
return new Alarm(alarmId, name, description, expression, state, actionsEnabled,
return new Alarm(alarmId, name, description, expression, state, enabled,
alarmActions == null ? alarm.getAlarmActions() : alarmActions,
okActions == null ? alarm.getOkActions() : okActions,
undeterminedActions == null ? alarm.getUndeterminedActions() : undeterminedActions);
@ -167,21 +178,20 @@ public class AlarmService {
private void updateInternal(String tenantId, String alarmId, boolean patch, String name,
String description, String expression, AlarmExpression alarmExpression, AlarmState state,
Boolean actionsEnabled, List<String> alarmActions, List<String> okActions,
Boolean enabled, List<String> alarmActions, List<String> okActions,
List<String> undeterminedActions) {
Entry<Map<String, AlarmSubExpression>, Map<String, AlarmSubExpression>> subAlarms = oldAndNewSubExpressionsFor(
alarmId, alarmExpression);
SubExpressions subExpressions = subExpressionsFor(alarmId, alarmExpression);
try {
LOG.debug("Updating alarm {} for tenant {}", name, tenantId);
repo.update(tenantId, alarmId, patch, name, description, expression, state, actionsEnabled,
subAlarms.getKey().keySet(), subAlarms.getValue(), alarmActions, okActions,
undeterminedActions);
repo.update(tenantId, alarmId, patch, name, description, expression, state, enabled,
subExpressions.oldAlarmSubExpressions.keySet(), subExpressions.changedSubExpressions,
subExpressions.newAlarmSubExpressions, alarmActions, okActions, undeterminedActions);
// Notify interested parties of new alarm
// TODO pass changed sub alarms
// Notify interested parties of updated alarm
String event = Serialization.toJson(new AlarmUpdatedEvent(tenantId, alarmId, name,
expression, state, actionsEnabled, subAlarms.getKey(), null, subAlarms.getValue()));
expression, state, enabled, subExpressions.oldAlarmSubExpressions,
subExpressions.changedSubExpressions, subExpressions.newAlarmSubExpressions));
producer.send(new KeyedMessage<>(config.eventsTopic, tenantId, event));
} catch (Exception e) {
throw Exceptions.uncheck(e, "Error updating alarm for project / tenant %s", tenantId);
@ -189,26 +199,57 @@ public class AlarmService {
}
/**
* Returns an entry containing Maps of old and new sub expressions by comparing the
* Returns an entry containing Maps of old, changed, and new sub expressions by comparing the
* {@code alarmExpression} to the existing sub expressions for the {@code alarmId}.
*/
Entry<Map<String, AlarmSubExpression>, Map<String, AlarmSubExpression>> oldAndNewSubExpressionsFor(
String alarmId, AlarmExpression alarmExpression) {
Map<String, AlarmSubExpression> oldSubAlarms = repo.findSubExpressions(alarmId);
Set<AlarmSubExpression> oldSet = new HashSet<>(oldSubAlarms.values());
SubExpressions subExpressionsFor(String alarmId, AlarmExpression alarmExpression) {
BiMap<String, AlarmSubExpression> oldExpressions = HashBiMap.create(repo.findSubExpressions(alarmId));
Set<AlarmSubExpression> oldSet = oldExpressions.inverse().keySet();
Set<AlarmSubExpression> newSet = new HashSet<>(alarmExpression.getSubExpressions());
// Filter old sub expressions
Set<AlarmSubExpression> oldExpressions = Sets.difference(oldSet, newSet);
oldSubAlarms.values().retainAll(oldExpressions);
// Identify old or changed expressions
Set<AlarmSubExpression> oldOrChangedExpressions = new HashSet<>(Sets.difference(oldSet, newSet));
// Identify new sub expressions
Map<String, AlarmSubExpression> newSubAlarms = new HashMap<>();
Set<AlarmSubExpression> newExpressions = Sets.difference(newSet, oldSet);
for (AlarmSubExpression expression : newExpressions)
newSubAlarms.put(UUID.randomUUID().toString(), expression);
// Identify new or changed expressions
Set<AlarmSubExpression> newOrChangedExpressions = new HashSet<>(Sets.difference(newSet, oldSet));
return new SimpleEntry<>(oldSubAlarms, newSubAlarms);
// Find changed expressions
Map<String, AlarmSubExpression> changedExpressions = new HashMap<>();
for (Iterator<AlarmSubExpression> oldIt = oldOrChangedExpressions.iterator(); oldIt.hasNext();) {
AlarmSubExpression oldExpr = oldIt.next();
for (Iterator<AlarmSubExpression> newIt = newOrChangedExpressions.iterator(); newIt.hasNext();) {
AlarmSubExpression newExpr = newIt.next();
if (sameKeyFields(oldExpr, newExpr)) {
oldIt.remove();
newIt.remove();
changedExpressions.put(oldExpressions.inverse().get(oldExpr), newExpr);
}
}
}
// Remove old sub expressions
oldExpressions.values().retainAll(oldOrChangedExpressions);
// Create IDs for new expressions
Map<String, AlarmSubExpression> newExpressions = new HashMap<>();
for (AlarmSubExpression expression : newOrChangedExpressions)
newExpressions.put(UUID.randomUUID().toString(), expression);
SubExpressions subExpressions = new SubExpressions();
subExpressions.oldAlarmSubExpressions = oldExpressions;
subExpressions.changedSubExpressions = changedExpressions;
subExpressions.newAlarmSubExpressions = newExpressions;
return subExpressions;
}
/**
* Returns whether all of the fields of {@code a} and {@code b} are the same except the operator
* and threshold.
*/
private boolean sameKeyFields(AlarmSubExpression a, AlarmSubExpression b) {
return a.getMetricDefinition().equals(b.getMetricDefinition())
&& a.getFunction().equals(b.getFunction()) && a.getPeriod() == b.getPeriod()
&& a.getPeriods() == b.getPeriods();
}
/**
@ -231,4 +272,4 @@ public class AlarmService {
if (!notificationMethodRepo.exists(tenantId, action))
throw new InvalidEntityException("No notification method exists for action %s", action);
}
}
}

View File

@ -57,6 +57,7 @@ public interface AlarmRepository {
*/
void update(String tenantId, String id, boolean patch, String name, String description,
String expression, AlarmState state, boolean enabled, Collection<String> oldSubAlarmIds,
Map<String, AlarmSubExpression> changedSubAlarms,
Map<String, AlarmSubExpression> newSubAlarms, List<String> alarmActions,
List<String> okActions, List<String> undeterminedActions);
}

View File

@ -53,7 +53,7 @@ public class AlarmRepositoryImpl implements AlarmRepository {
id, tenantId, name, description, expression, AlarmState.UNDETERMINED.toString(), true);
// Persist sub-alarms
persistSubExpressions(h, id, subExpressions);
createSubExpressions(h, id, subExpressions);
// Persist actions
persistActions(h, id, AlarmState.ALARM, alarmActions);
@ -192,8 +192,9 @@ public class AlarmRepositoryImpl implements AlarmRepository {
@Override
public void update(String tenantId, String id, boolean patch, String name, String description,
String expression, AlarmState state, boolean actionsEnabled,
Collection<String> oldSubAlarmIds, Map<String, AlarmSubExpression> newSubAlarms,
List<String> alarmActions, List<String> okActions, List<String> undeterminedActions) {
Collection<String> oldSubAlarmIds, Map<String, AlarmSubExpression> changedSubAlarms,
Map<String, AlarmSubExpression> newSubAlarms, List<String> alarmActions,
List<String> okActions, List<String> undeterminedActions) {
Handle h = db.open();
try {
@ -207,8 +208,17 @@ public class AlarmRepositoryImpl implements AlarmRepository {
for (String oldSubAlarmId : oldSubAlarmIds)
h.execute("delete from sub_alarm where id = ?", oldSubAlarmId);
// Update changed sub-alarms
if (changedSubAlarms != null)
for (Map.Entry<String, AlarmSubExpression> entry : changedSubAlarms.entrySet()) {
AlarmSubExpression sa = entry.getValue();
h.execute(
"update sub_alarm set operator = ?, threshold = ?, updated_at = NOW() where id = ?",
sa.getOperator().name(), sa.getThreshold(), entry.getKey());
}
// Insert new sub-alarms
persistSubExpressions(h, id, newSubAlarms);
createSubExpressions(h, id, newSubAlarms);
// Delete old actions
if (patch) {
@ -274,7 +284,7 @@ public class AlarmRepositoryImpl implements AlarmRepository {
alarm.setUndeterminedActions(findActionsById(handle, alarm.getId(), AlarmState.UNDETERMINED));
}
private void persistSubExpressions(Handle handle, String id,
private void createSubExpressions(Handle handle, String id,
Map<String, AlarmSubExpression> alarmSubExpressions) {
if (alarmSubExpressions != null)
for (Map.Entry<String, AlarmSubExpression> subEntry : alarmSubExpressions.entrySet()) {

View File

@ -10,10 +10,10 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
@ -24,6 +24,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.hpcloud.mon.MonApiConfiguration;
import com.hpcloud.mon.app.AlarmService.SubExpressions;
import com.hpcloud.mon.app.command.UpdateAlarmCommand;
import com.hpcloud.mon.common.model.alarm.AlarmExpression;
import com.hpcloud.mon.common.model.alarm.AlarmState;
@ -120,24 +121,25 @@ public class AlarmServiceTest {
public void testOldAndNewSubExpressionsFor() {
Map<String, AlarmSubExpression> oldSubExpressions = new HashMap<>();
oldSubExpressions.put("444", AlarmSubExpression.of("avg(foo{instance_id=123}) > 90"));
oldSubExpressions.put("555", AlarmSubExpression.of("avg(bar{instance_id=777}) > 80"));
oldSubExpressions.put("111", AlarmSubExpression.of("avg(foo{instance_id=123}) > 1"));
oldSubExpressions.put("222", AlarmSubExpression.of("avg(foo{instance_id=456}) > 2"));
oldSubExpressions.put("333", AlarmSubExpression.of("avg(foo{instance_id=789}) > 3"));
when(repo.findSubExpressions(eq("123"))).thenReturn(oldSubExpressions);
String newExprStr = "avg(foo{instance_id=123}) > 90 or avg(bar{instance_id=xxxx}) > 10 or avg(baz{instance_id=654}) > 123";
String newExprStr = "avg(foo{instance_id=123}) > 1 or avg(foo{instance_id=456}) <= 22 or avg(foo{instance_id=444}) > 4";
AlarmExpression newExpr = AlarmExpression.of(newExprStr);
Entry<Map<String, AlarmSubExpression>, Map<String, AlarmSubExpression>> expressions = service.oldAndNewSubExpressionsFor(
"123", newExpr);
SubExpressions expressions = service.subExpressionsFor("123", newExpr);
// Assert old expressions
Map<String, AlarmSubExpression> expectedOldExprs = new HashMap<>();
expectedOldExprs.put("555", AlarmSubExpression.of("avg(bar{instance_id=777}) > 80"));
assertEquals(expressions.getKey(), expectedOldExprs);
assertEquals(expressions.oldAlarmSubExpressions,
Collections.singletonMap("333", AlarmSubExpression.of("avg(foo{instance_id=789}) > 3")));
// Assert changed expressions
assertEquals(expressions.changedSubExpressions,
Collections.singletonMap("222", AlarmSubExpression.of("avg(foo{instance_id=456}) <= 22")));
// Assert new expressions
assertTrue(expressions.getValue().containsValue(
AlarmSubExpression.of("avg(bar{instance_id=xxxx}) > 10")));
assertTrue(expressions.getValue().containsValue(
AlarmSubExpression.of("avg(baz{instance_id=654}) > 123")));
assertTrue(expressions.newAlarmSubExpressions.containsValue(AlarmSubExpression.of("avg(foo{instance_id=444}) > 4")));
}
}

View File

@ -79,12 +79,15 @@ public class AlarmRepositoryImplTest {
handle.execute("insert into alarm_action values ('123', 'ALARM', '77778687')");
handle.execute("insert into alarm (id, tenant_id, name, expression, state, actions_enabled, created_at, updated_at, deleted_at) "
+ "values ('234', 'bob', '50% CPU', 'avg(hpcs.compute{flavor_id=777, image_id=888, metric_name=mem}) > 20', 'UNDETERMINED', 1, NOW(), NOW(), NULL)");
+ "values ('234', 'bob', '50% CPU', 'avg(hpcs.compute{flavor_id=777, image_id=888, metric_name=mem}) > 20 and avg(hpcs.compute{flavor_id=777}) < 100', 'UNDETERMINED', 1, NOW(), NOW(), NULL)");
handle.execute("insert into sub_alarm (id, alarm_id, function, metric_name, operator, threshold, period, periods, state, created_at, updated_at) "
+ "values ('222', '234', 'avg', 'hpcs.compute', 'GT', 20, 60, 1, 'UNDETERMINED', NOW(), NOW())");
handle.execute("insert into sub_alarm (id, alarm_id, function, metric_name, operator, threshold, period, periods, state, created_at, updated_at) "
+ "values ('223', '234', 'avg', 'hpcs.compute', 'LT', 100, 60, 1, 'UNDETERMINED', NOW(), NOW())");
handle.execute("insert into sub_alarm_dimension values ('222', 'flavor_id', '777')");
handle.execute("insert into sub_alarm_dimension values ('222', 'image_id', '888')");
handle.execute("insert into sub_alarm_dimension values ('222', 'metric_name', 'mem')");
handle.execute("insert into sub_alarm_dimension values ('223', 'flavor_id', '777')");
handle.execute("insert into alarm_action values ('234', 'ALARM', '29387234')");
handle.execute("insert into alarm_action values ('234', 'ALARM', '77778687')");
}
@ -122,21 +125,27 @@ public class AlarmRepositoryImplTest {
beforeMethod();
List<String> oldSubAlarmIds = Arrays.asList("222");
AlarmSubExpression changedSubExpression = AlarmSubExpression.of("avg(hpcs.compute{flavor_id=777}) <= 200");
Map<String, AlarmSubExpression> changedSubExpressions = ImmutableMap.<String, AlarmSubExpression>builder()
.put("223", changedSubExpression)
.build();
AlarmSubExpression newSubExpression = AlarmSubExpression.of("avg(foo{flavor_id=777}) > 333");
Map<String, AlarmSubExpression> newSubExpressions = ImmutableMap.<String, AlarmSubExpression>builder()
.put("555", newSubExpression)
.build();
repo.update("bob", "234", false, "90% CPU", null, "avg(foo{flavor_id=777}) > 333",
AlarmState.ALARM, false, oldSubAlarmIds, newSubExpressions, alarmActions, null, null);
repo.update("bob", "234", false, "90% CPU", null, "avg(foo{flavor_id=777}) > 333 and avg(hpcs.compute{flavor_id=777}) <= 200",
AlarmState.ALARM, false, oldSubAlarmIds, changedSubExpressions, newSubExpressions,
alarmActions, null, null);
Alarm alarm = repo.findById("bob", "234");
Alarm expected = new Alarm("234", "90% CPU", null, "avg(foo{flavor_id=777}) > 333",
Alarm expected = new Alarm("234", "90% CPU", null, "avg(foo{flavor_id=777}) > 333 and avg(hpcs.compute{flavor_id=777}) <= 200",
AlarmState.ALARM, false, alarmActions, Collections.<String>emptyList(),
Collections.<String>emptyList());
assertEquals(expected, alarm);
Map<String, AlarmSubExpression> subExpressions = repo.findSubExpressions("234");
assertEquals(subExpressions.get("223"), changedSubExpression);
assertEquals(subExpressions.get("555"), newSubExpression);
}
@ -216,15 +225,20 @@ public class AlarmRepositoryImplTest {
public void shouldFind() {
List<Alarm> alarms = repo.find("bob");
assertEquals(alarms, Arrays.asList(
new Alarm("123", "90% CPU", null,
"avg(hpcs.compute{flavor_id=777, image_id=888, metric_name=cpu, device=1}) > 10",
AlarmState.UNDETERMINED, true, Arrays.asList("29387234", "77778687"),
Collections.<String>emptyList(), Collections.<String>emptyList()),
new Alarm("234", "50% CPU", null,
"avg(hpcs.compute{flavor_id=777, image_id=888, metric_name=mem}) > 20",
AlarmState.UNDETERMINED, true, Arrays.asList("29387234", "77778687"),
Collections.<String>emptyList(), Collections.<String>emptyList())));
assertEquals(
alarms,
Arrays.asList(
new Alarm("123", "90% CPU", null,
"avg(hpcs.compute{flavor_id=777, image_id=888, metric_name=cpu, device=1}) > 10",
AlarmState.UNDETERMINED, true, Arrays.asList("29387234", "77778687"),
Collections.<String>emptyList(), Collections.<String>emptyList()),
new Alarm(
"234",
"50% CPU",
null,
"avg(hpcs.compute{flavor_id=777, image_id=888, metric_name=mem}) > 20 and avg(hpcs.compute{flavor_id=777}) < 100",
AlarmState.UNDETERMINED, true, Arrays.asList("29387234", "77778687"),
Collections.<String>emptyList(), Collections.<String>emptyList())));
}
public void shouldDeleteById() {