Refine logging

Change-Id: Idec004cb497e20daa58a5ed6f0075d097de6fcb9

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/AMPLGenerator.java

@@ -152,6 +152,10 @@ public class AMPLGenerator {
     /**
      * Calculate all metrics that are actually used.
      *
+     * NOTE: may also contain variable names.  This is not a problem as long
+     * as we use the result of this method to filter out unused metrics, but
+     * must be fixed if we use this method for other purposes as well.
+     *
      * @param app the NebulousApp.
      * @return The set of raw or composite metrics that are used in
      *  performance indicators, constraints or utility functions.
@@ -162,6 +166,7 @@ public class AMPLGenerator {
         Set<String> result = new HashSet<>();
         // collect from performance indicators
         for (final JsonNode indicator : app.getPerformanceIndicators().values()) {
+            // FIXME: note that here we collect also variables
             indicator.withArray("arguments").elements()
                 .forEachRemaining(node -> result.add(node.asText()));
         }

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java

@@ -128,6 +128,7 @@ public class ExnConnector {
                 NebulousApp app = NebulousApp.newFromAppMessage(mapper.valueToTree(body), amplMessagePublisher);
                 NebulousApps.add(app);
                 app.sendAMPL();
+                app.deployUnmodifiedApplication();
             } catch (Exception e) {
                 log.error("Error while receiving app creation message", e);
             }

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/LocalExecution.java

@@ -51,6 +51,7 @@ public class LocalExecution implements Callable<Integer> {
         if (connector != null) {
             log.debug("Sending AMPL to channel {}", publisher);
             app.sendAMPL();
+            app.deployUnmodifiedApplication();
         }
         System.out.println(AMPLGenerator.generateAMPL(app));
         // TODO: wait for solver reply here?

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/Main.java

@@ -154,7 +154,7 @@ public class Main implements Callable<Integer> {
                 log.warn("Trying to use a file as log directory, or directory not writable: {}. Continuing without file logging.", logDirectory);
                 logDirectory = null;
             } else {
-                log.debug("Logging all messages to directory {}", logDirectory);
+                log.info("Logging all messages to directory {}", logDirectory);
             }
         }
         // Start connection to SAL if possible.
@@ -171,7 +171,7 @@ public class Main implements Callable<Integer> {
         }
         // Start connection to ActiveMQ if possible.
         if (activemq_user != null && activemq_password != null) {
-            log.debug("Preparing ActiveMQ connection: host={} port={}",
+            log.info("Preparing ActiveMQ connection: host={} port={}",
                 activemq_host, activemq_port);
             activeMQConnector
               = new ExnConnector(activemq_host, activemq_port,
@@ -203,6 +203,7 @@ public class Main implements Callable<Integer> {
         // Note that we try to synchronize, even if we didn't connect to
         // ActiveMQ.  This is so that the container can be deployed.  (If the
         // container terminates, the build registers as unsuccessful.)
+        log.info("Optimiser-controller Initialization complete, waiting for messages");
         try {
             exn_synchronizer.await();
         } catch (InterruptedException e) {

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousApp.java

@@ -185,6 +185,7 @@ public class NebulousApp {
                 log.error("Could not find kubevela or parameters in app creation message.");
                 return null;
             } else {
+                Main.logFile("incoming-kubevela-" + app_message.at(uuid_path).textValue() + ".yaml", kubevela_string);
                 return new NebulousApp(app_message,
                     (ObjectNode)readKubevelaString(kubevela_string),
                     ampl_message_channel);
@@ -342,8 +343,8 @@ public class NebulousApp {
         }
 
         ampl_message_channel.send(mapper.convertValue(msg, Map.class), getUUID(), true);
-        Main.logFile(getUUID() + "to-solver.json", msg.toString());
+        Main.logFile("to-solver-" + getUUID() + ".json", msg.toString());
-        Main.logFile(getUUID() + ".ampl", ampl);
+        Main.logFile("to-solver-" + getUUID() + ".ampl", ampl);
     }
 
     /**
@@ -365,7 +366,7 @@ public class NebulousApp {
         return "";
     }
 
-	/**
+    /**
      * Handle incoming solver message.
      *
      * @param solution The message from the solver, containing a field
@@ -399,4 +400,11 @@ public class NebulousApp {
         }
     }
 
+    /**
+     * Deploy an application, bypassing the solver.  Will deploy unmodified
+     * KubeVela, as given by the initial app creation message.
+     */
+    public void deployUnmodifiedApplication() {
+        NebulousAppDeployer.deployApplication(original_kubevela, UUID, name);
+    }
 }

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java

@@ -20,9 +20,12 @@ import org.ow2.proactive.sal.model.Requirement;
 import org.ow2.proactive.sal.model.RequirementOperator;
 import org.ow2.proactive.sal.model.TaskDefinition;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -43,6 +46,8 @@ public class NebulousAppDeployer {
     @Getter
     private static CommandsInstallation controllerInstallation = new CommandsInstallation();
 
+    private static final ObjectMapper yaml_mapper = new ObjectMapper(new YAMLFactory());
+
     // TODO: find out the commands to initialize the workers
     /**
      * The installation scripts to send to SAL for a NebulOuS worker node.
@@ -184,8 +189,8 @@ public class NebulousAppDeployer {
                         RequirementOperator.GEQ, sal_memory));
                 }
             }
-            for (final JsonNode t : c.withArray("traits")) {
+            for (final JsonNode t : c.withArray("/traits")) {
-                // Check for node affinity / geoLocation / country
+                // TODO: Check for node affinity / geoLocation / country
             }
             // Finally, add requirements for this job to the map
             result.put(componentName, reqs);
@@ -266,6 +271,9 @@ public class NebulousAppDeployer {
         Map<String, List<Requirement>> requirements = getSalRequirementsFromKubevela(kubevela);
         Map<String, Integer> nodeCounts = getNodeCountFromKubevela(kubevela);
 
+        Main.logFile("node-requirements-" + appUUID + ".txt", requirements);
+        Main.logFile("node-counts-" + appUUID + ".txt", nodeCounts);
+
         // ------------------------------------------------------------
         // 2. Create SAL job
         log.debug("Creating job info for {}", appUUID);
@@ -331,31 +339,38 @@ public class NebulousAppDeployer {
         // ------------------------------------------------------------
         // 6. Create worker nodes from requirements
         log.debug("Starting worker nodes for {}", appUUID);
-        for (Map.Entry<String, List<Requirement>> e : requirements.entrySet()) {
+        // for (Map.Entry<String, List<Requirement>> e : requirements.entrySet()) {
-            List<NodeCandidate> candidates = NebulousApp.getSalConnector().findNodeCandidates(e.getValue());
+        //     List<NodeCandidate> candidates = NebulousApp.getSalConnector().findNodeCandidates(e.getValue());
-            if (candidates.isEmpty()) {
+        //     if (candidates.isEmpty()) {
-                log.error("Could not find node candidates for requirements: {}", e.getValue());
+        //         log.error("Could not find node candidates for requirements: {}", e.getValue());
-                return;
+        //         return;
-            }
+        //     }
-            NodeCandidate candidate = candidates.get(0);
+        //     NodeCandidate candidate = candidates.get(0);
-            // Here we specify the node names that we (hope to) use for node
+        //     // Here we specify the node names that we (hope to) use for node
-            // affinity declarations in KubeVela
+        //     // affinity declarations in KubeVela
-            IaasDefinition def = new IaasDefinition(
+        //     IaasDefinition def = new IaasDefinition(
-                e.getKey(), "nebulous-worker", candidate.getId(), candidate.getCloud().getId()
+        //         e.getKey(), "nebulous-worker", candidate.getId(), candidate.getCloud().getId()
-            );
+        //     );
-            int n = nodeCounts.get(e.getKey());
+        //     int n = nodeCounts.get(e.getKey());
-            log.debug("Asking for {} copies of {} for application {}", n, candidate, appUUID);
+        //     log.debug("Asking for {} copies of {} for application {}", n, candidate, appUUID);
-            success = NebulousApp.getSalConnector().addNodes(Collections.nCopies(n, def), appUUID);
+        //     success = NebulousApp.getSalConnector().addNodes(Collections.nCopies(n, def), appUUID);
-            if (!success) {
+        //     if (!success) {
-                log.error("Failed to add node: {}", candidate);
+        //         log.error("Failed to add node: {}", candidate);
-            }
+        //     }
-        }
+        // }
 
         // ------------------------------------------------------------
         // 7. Rewrite KubeVela file, based on running node names
 
         // TODO
         JsonNode rewritten = addNodeAffinities(kubevela);
+        String rewritten_kubevela = "---\n# Did not manage to create rewritten KubeVela";
+	try {
+	    rewritten_kubevela = yaml_mapper.writeValueAsString(rewritten);
+	} catch (JsonProcessingException e) {
+            log.error("Failed to convert KubeVela to YAML; this should never happen", e);
+	}
+        Main.logFile("rewritten-kubevela-" + appUUID + ".yaml", rewritten_kubevela);
 
         // ------------------------------------------------------------
         // 8. Submit KubeVela file to coordinator node

optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/SalConnector.java

@@ -44,7 +44,7 @@ import java.util.List;
 public class SalConnector {
 
     private static final String connectStr = "sal/pagateway/connect";
-    private static final String getAllCloudsStr = "sal/cloud";
+    private static final String getAllCloudsStr = "sal/clouds";
     private static final String findNodeCandidatesStr = "sal/nodecandidates";
     private static final String createJobStr = "sal/job";
     private static final String getJobsStr = "sal/job"; // same, but different method/body