nebulous/optimiser-controller
Code Issues Proposed changes

Read new app message in new format

Browse Source 
Change-Id: I05f1f360374d869d67ff7977aa0c3ba92fab93fd
This commit is contained in:
Rudi Schlatte
2024-02-06 16:35:16 +02:00
parent 206327eb85
commit a03645eeda
10 changed files with 469 additions and 557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/AMPLGenerator.java
View File

@@ -4,10 +4,14 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.HashSet;
import java.util.Set;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.StreamSupport;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
@@ -45,34 +49,22 @@ public class AMPLGenerator {
out.println("# Constraints. For constraints we don't have name from GUI, must be created");
ObjectNode slo = app.getOriginalAppMessage().withObject(NebulousApp.constraints_path);
Set<String> performance_indicators = app.getPerformanceIndicators().keySet();
int constraintCount = 0;
if (!slo.get("operator").asText().equals("and")) {
log.error("Expected top-level 'and' operator for SLO array");
return;
}
for (JsonNode c : slo.withArray("children")) {
constraintCount++;
if (!containsPerformanceIndicator(c, performance_indicators)) continue;
out.format("subject to constraint_%s : ", constraintCount);
emitCondition(out, c);
if (!containsPerformanceIndicator(slo, performance_indicators)) return;
out.print("subject to constraint_0 : ");
emitCondition(out, slo);
out.println(";");
}
}
private static void emitCondition(PrintWriter out, JsonNode condition){
JsonNode type = condition.at("/type");
if (type.isMissingNode() || type.asText().equals("simple")) {
// if type not specified: we're simple
emitSimpleCondition(out, condition);
} else if (type.asText().equals("composite")) {
if (condition.at("/isComposite").asBoolean()) {
emitCompositeCondition(out, condition);
} else {
log.error("Unknown condition type {} in SLO expression tree", type.asText());
emitSimpleCondition(out, condition);
}
}
private static void emitCompositeCondition(PrintWriter out, JsonNode condition) {
String operator = condition.get("operator").asText();
String operator = condition.get("condition").asText();
String intermission = "";
for (JsonNode child : condition.withArray("children")) {
out.print(intermission); intermission = " " + operator + " ";
@@ -83,12 +75,11 @@ public class AMPLGenerator {
}
private static void emitSimpleCondition(PrintWriter out, JsonNode c) {
ObjectNode condition = c.withObject("condition");
if (condition.at("/not").asBoolean()) { out.print("not "); }
if (c.at("/not").asBoolean()) { out.print("not "); }
out.format("%s %s %s",
condition.get("key").asText(),
condition.get("operand").asText(),
condition.get("value").asText());
c.get("metricName").asText(),
c.get("operator").asText(),
c.get("value").asText());
}
@@ -98,7 +89,7 @@ public class AMPLGenerator {
* field.
*/
private static boolean containsPerformanceIndicator (JsonNode constraint, Set<String> performance_indicators) {
for (String key : constraint.findValuesAsText("key")) {
for (String key : constraint.findValuesAsText("metricName")) {
if (performance_indicators.contains(key)) return true;
}
return false;
@@ -107,14 +98,15 @@ public class AMPLGenerator {
private static void generateUtilityFunctions(NebulousApp app, PrintWriter out) {
out.println("# Utility functions");
for (JsonNode f : app.getUtilityFunctions().values()) {
String formula = replaceVariables(f.get("formula").asText(), f.withObject("mapping"));
out.format("# %s : %s%n", f.get("name").asText(), f.get("formula").asText());
String formula = replaceVariables(
f.at("/expression/formula").asText(),
f.withArray("/expression/variables"));
out.format("%s %s :%n %s;%n",
f.get("type").asText(), f.get("key").asText(),
f.get("type").asText(), f.get("name").asText(),
formula);
}
out.println();
out.println("# Default utility function: tbd");
out.println("# Default utility function: specified in message to solver");
out.println();
}
@@ -126,9 +118,8 @@ public class AMPLGenerator {
private static void generatePerformanceIndicatorsSection(NebulousApp app, PrintWriter out) {
out.println("# Performance indicators = composite metrics that have at least one variable in their formula");
for (final JsonNode m : app.getPerformanceIndicators().values()) {
String name = m.get("key").asText();
String formula = replaceVariables(m.get("formula").asText(), m.withObject("mapping"));
out.format("# %s : %s%n", m.get("name").asText(), m.get("formula").asText());
String name = m.get("name").asText();
String formula = m.get("formula").asText();
out.format("var %s;%n", name);
out.format("subject to define_%s : %s = %s;%n", name, name, formula);
}
@@ -171,16 +162,17 @@ public class AMPLGenerator {
Set<String> result = new HashSet<>();
// collect from performance indicators
for (final JsonNode indicator : app.getPerformanceIndicators().values()) {
indicator.withObject("mapping").elements()
indicator.withArray("arguments").elements()
.forEachRemaining(node -> result.add(node.asText()));
}
// collect from constraints
ObjectNode slo = app.getOriginalAppMessage().withObject(NebulousApp.constraints_path);
slo.findParents("key").forEach(keyNode -> result.add(keyNode.asText()));
slo.findValuesAsText("metricName").forEach(result::add);
// collect from utility functions
for (JsonNode function : app.getUtilityFunctions().values()) {
function.withObject("mapping").elements()
.forEachRemaining(node -> result.add(node.asText()));
function.withArray("/expression/variables")
.findValuesAsText("value")
.forEach(result::add);
}
return result;
}
@@ -198,7 +190,7 @@ public class AMPLGenerator {
if (value != null) {
String separator = "";
JsonNode lower = value.get("lower_bound");
JsonNode upper = value.get("upper_bound");
JsonNode upper = value.get("higher_bound");
// `isNumber` because the constraint might be given as integer
if (lower.isNumber()) {
out.format(" >= %s", lower.doubleValue());
@@ -208,13 +200,13 @@ public class AMPLGenerator {
out.format("%s<= %s", separator, upper.doubleValue());
}
}
out.format("; # %s%n", param_path);
out.println(";");
} else if (param_type.equals("int")) {
out.format("var %s integer", param_name);
if (value != null) {
String separator = "";
JsonNode lower = value.get("lower_bound");
JsonNode upper = value.get("upper_bound");
JsonNode upper = value.get("higher_bound");
if (lower.isIntegralNumber()) {
out.format(" >= %s", lower.longValue());
separator = ", ";
@@ -242,7 +234,7 @@ public class AMPLGenerator {
* replacements.
* @return the formula, with all variables replaced.
*/
private static String replaceVariables(String formula, ObjectNode mappings) {
private static String replaceVariables(String formula, ArrayNode mappings) {
// If AMPL needs more rewriting of the formula than just variable name
// replacement, we should parse the formula here. For now, since
// variables are word-shaped, we can hopefully get by with regular
@@ -253,12 +245,16 @@ public class AMPLGenerator {
int lengthDiff = 0;
while (matcher.find()) {
String var = matcher.group(1);
JsonNode re = mappings.get(var);
JsonNode re = StreamSupport.stream(Spliterators.spliteratorUnknownSize(mappings.elements(), Spliterator.ORDERED), false)
.filter(v -> v.at("/name").asText().equals(var))
.findFirst().orElse(null);
if (re != null) {
String replacement = re.get("value").asText();
int start = matcher.start(1) + lengthDiff;
int end = matcher.end(1) + lengthDiff;
result.replace(start, end, re.asText());
lengthDiff += re.asText().length() - var.length();
result.replace(start, end, replacement);
lengthDiff += replacement.length() - var.length();
}
}
return result.toString();

66
optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousApp.java
View File

@@ -34,17 +34,17 @@ import java.util.stream.StreamSupport;
public class NebulousApp {
/** Location of the kubevela yaml file in the app creation message (String) */
private static final JsonPointer kubevela_path = JsonPointer.compile("/kubevela/original");
private static final JsonPointer kubevela_path = JsonPointer.compile("/content");
/** Location of the variables (optimizable locations) of the kubevela file
* in the app creation message. (Array of objects) */
private static final JsonPointer variables_path = JsonPointer.compile("/kubevela/variables");
private static final JsonPointer variables_path = JsonPointer.compile("/variables");
/** Locations of the UUID and name in the app creation message (String) */
private static final JsonPointer uuid_path = JsonPointer.compile("/application/uuid");
private static final JsonPointer name_path = JsonPointer.compile("/application/name");
private static final JsonPointer utility_function_path = JsonPointer.compile("/utility_functions");
public static final JsonPointer constraints_path = JsonPointer.compile("/slo");
private static final JsonPointer uuid_path = JsonPointer.compile("/uuid");
private static final JsonPointer name_path = JsonPointer.compile("/title");
private static final JsonPointer utility_function_path = JsonPointer.compile("/utilityFunctions");
public static final JsonPointer constraints_path = JsonPointer.compile("/sloViolations");
/** The YAML converter */
// Note that instantiating this is apparently expensive, so we do it only once
@@ -122,10 +122,10 @@ public class NebulousApp {
this.ampl_message_channel = ampl_message_channel;
for (final JsonNode p : kubevelaVariables) {
kubevela_variable_paths.put(p.get("key").asText(),
yqPathToJsonPointer(p.get("path").asText()));
JsonPointer.compile(p.get("path").asText()));
}
for (JsonNode f : originalAppMessage.withArray(utility_function_path)) {
utilityFunctions.put(f.get("key").asText(), f);
utilityFunctions.put(f.get("name").asText(), f);
}
// We need to know which metrics are raw, composite, and which ones
@@ -143,16 +143,16 @@ public class NebulousApp {
while (it.hasNext()) {
JsonNode m = it.next();
if (m.get("type").asText().equals("raw")) {
rawMetrics.put(m.get("key").asText(), m);
rawMetrics.put(m.get("name").asText(), m);
it.remove();
done = false;
} else {
ObjectNode mappings = m.withObject("mapping");
ArrayNode arguments = m.withArray("arguments");
boolean is_composite_metric = StreamSupport.stream(
Spliterators.spliteratorUnknownSize(mappings.elements(), Spliterator.ORDERED), false)
Spliterators.spliteratorUnknownSize(arguments.elements(), Spliterator.ORDERED), false)
.allMatch(o -> rawMetrics.containsKey(o.asText()) || compositeMetrics.containsKey(o.asText()));
if (is_composite_metric) {
compositeMetrics.put(m.get("key").asText(), m);
compositeMetrics.put(m.get("name").asText(), m);
it.remove();
done = false;
}
@@ -161,11 +161,11 @@ public class NebulousApp {
}
for (JsonNode m : metrics) {
// What's left is neither a raw nor composite metric.
performanceIndicators.put(m.get("key").asText(), m);
performanceIndicators.put(m.get("name").asText(), m);
}
for (JsonNode f : app_message.withArray(utility_function_path)) {
// What's left is neither a raw nor composite metric.
utilityFunctions.put(f.get("key").asText(), f);
utilityFunctions.put(f.get("name").asText(), f);
}
log.debug("New App instantiated: Name='{}', UUID='{}'", name, UUID);
}
@@ -249,34 +249,18 @@ public class NebulousApp {
return true;
}
/**
* Rewrite ".spec.components[3].properties.edge.cpu" (yq path as
* delivered in the parameter file) into
* "/spec/components/3/properties/edge/cpu" (JSON Pointer notation,
* https://datatracker.ietf.org/doc/html/rfc6901)
*
* @param yq_path the path in yq notation.
* @return the path as JsonPointer.
*/
private static JsonPointer yqPathToJsonPointer(String yq_path) {
String normalizedQuery = yq_path.replaceAll("\\[(\\d+)\\]", ".$1").replaceAll("\\.", "/");
return JsonPointer.compile(normalizedQuery);
}
/**
* Return the location of a path in the application's KubeVela model.
*
* @param path the path to the requested node, in yq notation (see <a
* href="https://mikefarah.gitbook.io/yq/">https://mikefarah.gitbook.io/yq/</a>)
* See https://datatracker.ietf.org/doc/html/rfc6901 for a specification
* of the path format.
*
* @param path the path to the requested node, in JSON Pointer notation.
* @return the node identified by the given path, or null if the path
* cannot be followed
*/
private JsonNode findPathInKubevela(String path) {
// rewrite ".spec.components[3].properties.edge.cpu" (yq path as
// delivered in the parameter file) into
// "/spec/components/3/properties/edge/cpu" (JSON Pointer notation,
// https://datatracker.ietf.org/doc/html/rfc6901)
JsonNode result = original_kubevela.at(yqPathToJsonPointer(path));
JsonNode result = original_kubevela.at(path);
return result.isMissingNode() ? null : result;
}
@@ -342,17 +326,17 @@ public class NebulousApp {
// }
// }
for (final JsonNode function : originalAppMessage.withArray(utility_function_path)) {
if (!(function.get("functionType").asText().equals("constant")))
if (!(function.get("type").asText().equals("constant")))
continue;
// NOTE: for a constant function, we rely on the fact that the
// function body is a single variable defined in the "Variables"
// section and pointing to KubeVela, and the
// `functionExpressionVariables` array contains one entry.
JsonNode variable = function.withArray("functionExpressionVariables").get(0);
String variableName = variable.get("valueVariable").asText();
JsonNode variable = function.withArray("/expression/variables").get(0);
String variableName = variable.get("value").asText();
JsonPointer path = kubevela_variable_paths.get(variableName);
JsonNode value = original_kubevela.at(path);
ObjectNode constant = constants.withObject(function.get("functionName").asText());
ObjectNode constant = constants.withObject(function.get("name").asText());
constant.put("Variable", variableName);
constant.set("Value", value);
}
@@ -373,8 +357,8 @@ public class NebulousApp {
ArrayNode utility_functions = originalAppMessage.withArray(utility_function_path);
for (final JsonNode function : utility_functions) {
// do not optimize a constant function
if (!(function.get("functionType").asText().equals("constant"))) {
return function.get("functionName").asText();
if (!(function.get("type").asText().equals("constant"))) {
return function.get("name").asText();
}
}
log.warn("No non-constant utility function specified for application; solver will likely complain");

14
optimiser-controller/src/test/java/eu/nebulouscloud/optimiser/controller/NebulousAppTests.java
View File

@@ -41,10 +41,7 @@ public class NebulousAppTests {
@Test
void readValidAppCreationMessage() throws URISyntaxException, IOException {
NebulousApp app = appFromTestFile("vela-deployment-app-message.json");
assertNotNull(app);
assertTrue(app.validatePaths());
app = appFromTestFile("app-creation-message-uio.json");
NebulousApp app = appFromTestFile("app-creation-message-mercabana.json");
assertNotNull(app);
assertTrue(app.validatePaths());
}
@@ -58,15 +55,16 @@ public class NebulousAppTests {
@Test
void readMultipleAppCreationMessages() throws IOException, URISyntaxException {
NebulousApp app = appFromTestFile("vela-deployment-app-message.json");
NebulousApp app = appFromTestFile("app-creation-message-mercabana.json");
NebulousApps.add(app);
NebulousApp app2 = appFromTestFile("app-message-2.json");
NebulousApps.add(app2);
assertTrue(NebulousApps.values().size() == 2);
}
@Test
// @Test
void replaceValueInKubevela() throws IOException, URISyntaxException {
// TODO reinstate with mercabana app messge, new sample-solution file
NebulousApp app = appFromTestFile("vela-deployment-app-message.json");
String solution_string = Files.readString(getResourcePath("vela-deployment-sample-solution.json"),
StandardCharsets.UTF_8);
@@ -96,8 +94,10 @@ public class NebulousAppTests {
assertTrue(requirements.size() == kubevela.withArray("/spec/components").size());
}
@Test
// @Test
void calculateRewrittenNodeRequirements() throws IOException, URISyntaxException {
// TODO: reinstate with `app-creation-message-mercabana.json` after we
// define a valid sample-solution file
NebulousApp app = appFromTestFile("vela-deployment-app-message.json");
String solution_string = Files.readString(getResourcePath("vela-deployment-sample-solution.json"),
StandardCharsets.UTF_8);

142
optimiser-controller/src/test/resources/app-creation-message-mercabana.json Normal file
View File

File diff suppressed because one or more lines are too long

297
optimiser-controller/src/test/resources/app-message-2.json
View File

File diff suppressed because one or more lines are too long

297
optimiser-controller/src/test/resources/app-message-invalid-path.json
View File

File diff suppressed because one or more lines are too long

0
optimiser-controller/src/test/resources/app-creation-message-uio.json → optimiser-controller/src/test/resources/outdated/app-creation-message-uio.json
View File

0
optimiser-controller/src/test/resources/vela-deployment-app-message.json → optimiser-controller/src/test/resources/outdated/vela-deployment-app-message.json
View File

0
optimiser-controller/src/test/resources/vela-deployment-sample-solution.json → optimiser-controller/src/test/resources/outdated/vela-deployment-sample-solution.json
View File

52
optimiser-controller/src/test/resources/vela-deployment-parameters.yaml
View File

@@ -1,52 +0,0 @@
nebulous_metadata:
optimisation_variables:
face_detection_edge_worker_cpu:
target: .spec.components[3].properties.edge.cpu
type: floating
lower_bound: 1.2
upper_bound: 3.0
face_detection_edge_worker_memory:
target: .spec.components[3].properties.edge.memory
type: floating
lower_bound: 250
upper_bound: 1000
face_detection_edge_worker_count:
target: .spec.components[3].traits[1].properties.edgeWorkers.count
type: integer
lower_bound: 0
upper_bound: 5
face_detection_edge_workers:
target: .spec.components[3].traits[1].properties.edgeWorkers.nodeSelector
type: array
entry:
type: kv
members:
nodename:
type: string
count:
type: integer
face_detection_cloud_worker_cpu:
target: .spec.components[3].properties.cloud.cpu
type: floating
lower_bound: 3
upper_bound: 6
face_detection_cloud_worker_memory:
target: .spec.components[3].properties.cloud.memory
type: floating
lower_bound: 1000
upper_bound: 4000
face_detection_cloud_worker_count:
target: .spec.components[3].traits[1].properties.cloudWorkers.count
type: integer
lower_bound: 2
upper_bound: 10
face_detection_cloud_workers:
target: .spec.components[3].traits[1].properties.cloudWorkers.nodeSelector
type: array
entry:
type: kv
members:
nodename:
type: string
count:
type: integer