nebulous/optimiser-controller
Code Issues Proposed changes

Fix node affinities

Browse Source 
- use key "kubernetes.io/hostname" for match criteria
- during deployment, keep track of node names per component
- use them for hostname list in rewritten KubeVela file

Change-Id: Ib6fc25e1d22971bd9c5696886ca3e30532325234
This commit is contained in:
Rudi Schlatte
2024-02-08 12:30:38 +02:00
parent 47d803529c
commit 86a473a7d3
2 changed files with 110 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousApp.java
View File

@@ -33,14 +33,28 @@ import java.util.stream.StreamSupport;
*/
@Slf4j
public class NebulousApp {
/**
* The UUID of the app. This identifies a specific application's ActiveMQ
* messages, etc. Chosen by the UI, unique across a NebulOuS
* installation.
*/
@Getter
private String UUID;
/**
* The app name, a user-readable string. Not safe to assume that this is
* a unique value.
*/
@Getter private String name;
// ----------------------------------------
// App message parsing stuff
/** Location of the kubevela yaml file in the app creation message (String) */
private static final JsonPointer kubevela_path = JsonPointer.compile("/content");
/** Location of the variables (optimizable locations) of the kubevela file
* in the app creation message. (Array of objects) */
private static final JsonPointer variables_path = JsonPointer.compile("/variables");
/** Locations of the UUID and name in the app creation message (String) */
private static final JsonPointer uuid_path = JsonPointer.compile("/uuid");
private static final JsonPointer name_path = JsonPointer.compile("/title");
@@ -54,6 +68,33 @@ public class NebulousApp {
/** General-purpose object mapper */
private static final ObjectMapper mapper = new ObjectMapper();
// ----------------------------------------
// AMPL stuff
/** The array of KubeVela variables in the app message. */
@Getter private ArrayNode kubevelaVariables;
/** Map from AMPL variable name to location in KubeVela. */
private Map<String, JsonPointer> kubevela_variable_paths = new HashMap<>();
/** The app's raw metrics, a map from key to the defining JSON node. */
@Getter private Map<String, JsonNode> rawMetrics = new HashMap<>();
/** The app's composite metrics, a map from key to the defining JSON node. */
@Getter private Map<String, JsonNode> compositeMetrics = new HashMap<>();
/** The app's performance indicators, a map from key to the defining JSON node. */
@Getter private Map<String, JsonNode> performanceIndicators = new HashMap<>();
/** The app's utility functions; the AMPL solver will optimize for one of these. */
@Getter private Map<String, JsonNode> utilityFunctions = new HashMap<>();
/**
* The constraints that are actually relevant for the optimizer. If a
* constraint does not contain a variable, we cannot influence it via the
* solver, so it should not be included in the AMPL file.
*/
@Getter private Set<JsonNode> effectiveConstraints = new HashSet<>();
// ----------------------------------------
// Deployment stuff
/** The original app message. */
@Getter private JsonNode originalAppMessage;
private ObjectNode original_kubevela;
/**
* The active SAL connector, or null if we operate offline.
*
@@ -67,37 +108,16 @@ public class NebulousApp {
private static SalConnector salConnector;
/**
* The UUID of the app. This is the UUID that identifies a specific
* application's ActiveMQ messages.
* Map of component name to machine name(s) deployed for that component.
* Component names are defined in the KubeVela file. We assume that
* component names stay constant during redeployment, i.e., once an
* application is deployed, its KubeVela file will not change.
*
* @return the UUID of the app
* Note that this map does not include the master node, since this is not
* specified in KubeVela.
*/
@Getter
private String UUID;
/** The app name, a user-defined string. Not safe to assume that this is
* a unique value. */
@Getter private String name;
/** The original app message. */
@Getter private JsonNode originalAppMessage;
private ObjectNode original_kubevela;
/** The array of KubeVela variables in the app message. */
@Getter private ArrayNode kubevelaVariables;
/** Map from AMPL variable name to location in KubeVela. */
private Map<String, JsonPointer> kubevela_variable_paths = new HashMap<>();
/** The app's raw metrics, a map from key to the defining JSON node. */
@Getter private Map<String, JsonNode> rawMetrics = new HashMap<>();
/** The app's composite metrics, a map from key to the defining JSON node. */
@Getter private Map<String, JsonNode> compositeMetrics = new HashMap<>();
/** The app's performance indicators, a map from key to the defining JSON node. */
@Getter private Map<String, JsonNode> performanceIndicators = new HashMap<>();
/** The app's utility functions; the AMPL solver will optimize for one of these. */
@Getter private Map<String, JsonNode> utilityFunctions = new HashMap<>();
/**
* The constraints that are actually effective: if a constraint does not
* contain a variable, we cannot influence it via the solver
*/
@Getter private Set<JsonNode> effectiveConstraints = new HashSet<>();
private Map<String, Set<String>> componentMachineNames = new HashMap<>();
/** When an app gets deployed or redeployed, this is where we send the AMPL file */
private Publisher ampl_message_channel;
@@ -415,7 +435,7 @@ public class NebulousApp {
// 2. Tell SAL to start all nodes, passing in the deployment
// scripts
// 3. Send KubeVela file for deployment
NebulousAppDeployer.deployApplication(kubevela, UUID, name);
NebulousAppDeployer.deployApplication(this, kubevela);
}
}
@@ -424,6 +444,6 @@ public class NebulousApp {
* KubeVela, as given by the initial app creation message.
*/
public void deployUnmodifiedApplication() {
NebulousAppDeployer.deployApplication(original_kubevela, UUID, name);
NebulousAppDeployer.deployApplication(this, original_kubevela);
}
}

91
optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java
View File

@@ -3,8 +3,12 @@ package eu.nebulouscloud.optimiser.controller;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.ow2.proactive.sal.model.AttributeRequirement;
import org.ow2.proactive.sal.model.CommandsInstallation;
@@ -199,12 +203,11 @@ public class NebulousAppDeployer {
}
/**
* Add affinities trait to all components.
* Add affinities trait to all components, except for those with a replica
* count of 0.
*
* TODO: we need to find out which key to use to get the node labels, as
* assigned by the SAL `addNodes` endpoint.
*
* #+begin_src yaml
* <pre>
* {@code
* traits:
* - type: affinity
* properties:
@@ -212,27 +215,36 @@ public class NebulousAppDeployer {
* required:
* nodeSelectorTerms:
* - matchExpressions:
* - key: label
* - key: "kubernetes.io/hostname"
* operator: In
* values: ["machinelabel"]
* #+end_src
* values: ["componentname-1", "componentname-2"]
* }</pre>
*
* @param kubevela the KubeVela specification to modify. This parameter is
* not modified.
* @param componentMachineNames Map from component name to node names
* where that component should be deployed.
* @return a fresh KubeVela specification with added nodeAffinity traits.
*/
public static JsonNode addNodeAffinities(JsonNode kubevela) {
public static JsonNode addNodeAffinities(JsonNode kubevela, Map<String, Set<String>> componentMachineNames) {
JsonNode result = kubevela.deepCopy();
for (final JsonNode c : result.withArray("/spec/components")) {
if (componentMachineNames.getOrDefault(c.get("name").asText(), Set.of()).isEmpty()){
// Do not generate trait at all if we didn't deploy any
// machines. This happens if replicas is 0
continue;
}
ArrayNode traits = c.withArray("traits");
ObjectNode trait = traits.addObject();
trait.put("type", "affinity");
ArrayNode nodeSelectorTerms = trait.withArray("/properties/nodeAffinity/required/nodeSelectorTerms");
ArrayNode matchExpressions = nodeSelectorTerms.addObject().withArray("matchExpressions");
ObjectNode term = matchExpressions.addObject();
term.put("key", "label")
term.put("key", "kubernetes.io/hostname")
.put("operator", "In");
term.withArray("values").add(c.get("name").asText());
componentMachineNames
.getOrDefault(c.get("name").asText(), Set.of())
.forEach(nodename -> term.withArray("values").add(nodename));
}
return result;
}
@@ -241,11 +253,12 @@ public class NebulousAppDeployer {
* Given a KubeVela file, extract node requirements, create the job, start
* its nodes and submit KubeVela.
*
* @param app the NebulOuS app object.
* @param kubevela the KubeVela file to deploy.
* @param appUUID the application UUID.
* @param appName the application name.
*/
public static void deployApplication(JsonNode kubevela, String appUUID, String appName) {
public static void deployApplication(NebulousApp app, JsonNode kubevela) {
String appUUID = app.getUUID();
String appName = app.getName();
log.info("Starting initial deployment of {}", appUUID);
if (NebulousApp.getSalConnector() == null) {
log.warn("Tried to submit job, but do not have a connection to SAL");
@@ -339,31 +352,41 @@ public class NebulousAppDeployer {
// ------------------------------------------------------------
// 6. Create worker nodes from requirements
log.debug("Starting worker nodes for {}", appUUID);
// for (Map.Entry<String, List<Requirement>> e : requirements.entrySet()) {
// List<NodeCandidate> candidates = NebulousApp.getSalConnector().findNodeCandidates(e.getValue());
// if (candidates.isEmpty()) {
// log.error("Could not find node candidates for requirements: {}", e.getValue());
// return;
// }
// NodeCandidate candidate = candidates.get(0);
// // Here we specify the node names that we (hope to) use for node
// // affinity declarations in KubeVela
// IaasDefinition def = new IaasDefinition(
// e.getKey(), "nebulous-worker", candidate.getId(), candidate.getCloud().getId()
// );
// int n = nodeCounts.get(e.getKey());
// log.debug("Asking for {} copies of {} for application {}", n, candidate, appUUID);
// success = NebulousApp.getSalConnector().addNodes(Collections.nCopies(n, def), appUUID);
// if (!success) {
// log.error("Failed to add node: {}", candidate);
// }
// }
for (Map.Entry<String, List<Requirement>> e : requirements.entrySet()) {
String componentName = e.getKey();
int numberOfNodes = nodeCounts.get(e.getKey());
Set<String> nodeNames = new HashSet<>();
IntStream.rangeClosed(1, numberOfNodes)
.mapToObj(i -> nodeNames.add(String.format("%s-%s",
componentName, i)));
app.getComponentMachineNames().put(componentName, nodeNames);
if (numberOfNodes == 0) {
// Do not ask for node candidates if this component's replica
// count is 0. Note that we still set the app's machine names
// to an empty set.
continue;
}
List<NodeCandidate> candidates = NebulousApp.getSalConnector().findNodeCandidates(e.getValue());
if (candidates.isEmpty()) {
log.error("Could not find node candidates for requirements: {}", e.getValue());
return;
}
NodeCandidate candidate = candidates.get(0);
List<IaasDefinition> componentDefinitions = nodeNames.stream()
.map(name -> new IaasDefinition(name, "nebulous-worker", candidate.getId(), candidate.getCloud().getId()))
.collect(Collectors.toList());
log.debug("Asking for {} copies of {} for application {}", numberOfNodes, candidate, appUUID);
success = NebulousApp.getSalConnector().addNodes(componentDefinitions, appUUID);
if (!success) {
log.error("Failed to add nodes for component {}", componentName);
}
}
// ------------------------------------------------------------
// 7. Rewrite KubeVela file, based on running node names
// TODO
JsonNode rewritten = addNodeAffinities(kubevela);
JsonNode rewritten = addNodeAffinities(kubevela, app.getComponentMachineNames());
String rewritten_kubevela = "---\n# Did not manage to create rewritten KubeVela";
try {
rewritten_kubevela = yaml_mapper.writeValueAsString(rewritten);