import os import get_data as file import random import json from datetime import datetime import db.db_functions as db_functions # Boolean_Variables = ['Extend offered network capacity', 'Extend offered processing capacity', 'Extend offered memory capacity', # 'Fog resources addition', 'Edge resources addition', 'Solid State Drive'] Boolean_Variables = [ "fd871ec6-d953-430d-a354-f13c66fa8bc9", "dcedb196-2c60-4c29-a66d-0e768cfd698a", "0cf00a53-fd33-4887-bb38-e0bbb04e3f3e", "d95c1dae-1e22-4fb4-9cdc-743e96d0dddc", "8cd09fe9-c119-4ccd-b651-0f18334dbbe4", "7147995c-8e68-4106-ab24-f0a7673eb5f5", "c1c5b3c9-6178-4d67-a7e3-0285c2bf98ef"] # Used to extract_SAL_node_candidate_data from Use Side for DataGrid # def extract_SAL_node_candidate_data_Front(json_data): # default_criteria_list = ["cores", "ram", "disk", "memoryPrice", "price"] # # if isinstance(json_data, dict): # Single node dictionary # json_data = [json_data] # Wrap it in a list # # extracted_data = [] # node_ids = [] # node_names = [] # # for item in json_data: # hardware_info = item.get("hardware", {}) # # Extract default criteria values # default_criteria_values = {criteria: hardware_info.get(criteria, 0.0) if criteria in hardware_info else item.get(criteria, 0.0) for criteria in default_criteria_list} # # # Correctly extract the providerName from the cloud information # cloud_info = item.get("cloud", {}) # get the cloud info or default to an empty dict # api_info = cloud_info.get("api", {}) # provider_name = api_info.get("providerName", "Unknown Provider") # # # each item is now a dictionary # node_data = { # "nodeId": item.get("nodeId", ''), # "id": item.get('id', ''), # "nodeCandidateType": item.get("nodeCandidateType", ''), # **default_criteria_values, # Unpack default criteria values into node_data # "hardware": hardware_info, # "location": item.get("location", {}), # "image": item.get("image", {}), # "providerName": provider_name # } # extracted_data.append(node_data) # node_ids.append(node_data["id"]) # # # print("Before create_node_name") # node_names.append(create_node_name(node_data)) # call create_node_name function # # print("After create_node_name") # # return extracted_data, node_ids, node_names def extract_SAL_node_candidate_data_Front(json_data): default_criteria_list = ["cores", "ram", "disk", "memoryPrice", "price"] if isinstance(json_data, dict): # Single node dictionary json_data = [json_data] # Wrap it in a list extracted_data = [] node_ids = [] node_names = [] for item in json_data: hardware_info = item.get("hardware", {}) # Extract default criteria values default_criteria_values = {criteria: hardware_info.get(criteria, 0.0) if criteria in hardware_info else item.get(criteria, 0.0) for criteria in default_criteria_list} cloud_info = item.get("cloud", {}) # get the cloud info or default to an empty dict node_type = item.get("nodeCandidateType", "") # extract the providerName from the cloud information if node_type == "EDGE": provider_name = "-" # For "EDGE" type, set provider_name as "-" else: api_info = cloud_info.get("api", {}) provider_name = api_info.get("providerName", "Unknown Provider") # For other types, fetch from api_info # each item is now a dictionary node_data = { "nodeId": item.get("nodeId", ''), "id": item.get('id', ''), "nodeCandidateType": item.get("nodeCandidateType", ''), **default_criteria_values, # Unpack default criteria values into node_data "hardware": hardware_info, "location": item.get("location", {}), "image": item.get("image", {}), "providerName": provider_name } extracted_data.append(node_data) node_ids.append(node_data["id"]) # print("Before create_node_name") node_names.append(create_node_name(node_data)) # call create_node_name function # print("After create_node_name") return extracted_data, node_ids, node_names # Used to create node names for DataGrid def create_node_name(node_data): node_type = node_data.get("nodeCandidateType", "UNKNOWN_TYPE") # Initialize default values node_city = "" node_country = "" node_os_family = "Unknown OS" provider_name = node_data.get("providerName", "") # Safely access nested properties for city and country location = node_data.get("location") if location and "geoLocation" in location and location["geoLocation"]: geo_location = location["geoLocation"] node_city = geo_location.get("city", "") node_country = geo_location.get("country", "") image = node_data.get("image") if image and "operatingSystem" in image and image["operatingSystem"]: operating_system = image["operatingSystem"] node_os_family = operating_system.get("operatingSystemFamily", node_os_family) cores = node_data.get("cores", "") ram = node_data.get("ram", "") # Construct the node name with conditional inclusions node_name_parts = [node_type] if node_city and node_country: node_name_parts.append(f"{node_city}, {node_country}") if provider_name: node_name_parts.append(f"Provider: {provider_name}") node_name_parts.append(f"OS: {node_os_family}") if cores: node_name_parts.append(f"Cores: {cores} ") if ram: node_name_parts.append(f"RAM: {ram} ") node_name = " - ".join(part for part in node_name_parts if part) # Only include non-empty parts return node_name # Used to extract_SAL_node_candidate_data from App Side working with Optimizer def extract_SAL_node_candidate_data(json_string): # print("Entered in extract_SAL_node_candidate_data") json_data = json.loads(json_string) # try: # json_data = json.loads(json_string) # json_data is a list of dictionaries # except json.JSONDecodeError as e: # print(f"Error parsing JSON inside extract_SAL_node_candidate_data(): {e}") # return [], 0, [], [] extracted_data = [] for item in json_data: # Ensure each item is a dictionary before accessing it if isinstance(item, dict): node_data = { "nodeId": item.get("nodeId", ''), "id": item.get('id', ''), "nodeCandidateType": item.get("nodeCandidateType", ''), "price": item.get("price", ''), "pricePerInvocation": item.get("pricePerInvocation", ''), "memoryPrice": item.get("memoryPrice", ''), "hardware": item.get("hardware", {}) } extracted_data.append(node_data) else: print(f"Unexpected item format: {item}") number_of_nodes = len(extracted_data) node_ids = [node['id'] for node in extracted_data] node_names = [node['id'] for node in extracted_data] return extracted_data, node_ids, node_names # Used to map the criteria from SAL's response with the selected criteria (from frontend) def create_criteria_mapping(): field_mapping = { # "Cost": "price", "Operating cost": "price", "Memory Price": "memoryPrice", "Number of CPU Cores": "cores", "Memory Size": "ram", "Storage Capacity": "disk" } return field_mapping # Used to create the required structure for the Evaluation in process_evaluation_data endpoint def transform_grid_data_to_table(json_data): grid_data = json_data.get('gridData', []) relative_wr_data = json_data.get('relativeWRData', []) immediate_wr_data = json_data.get('immediateWRData', []) node_names = json_data.get('nodeNames', []) node_ids = [] # Initialize temporary dictionaries to organize the data temp_data_table = {} criteria_titles = [] # Mapping for ordinal values ordinal_value_mapping = {"High": 3, "Medium": 2, "Low": 1} boolean_value_mapping = {"True": 1, "False": 0} for node in grid_data: # node_name = node.get('name') node_ids.append(node.get('id')) node_id = node.get('id') criteria_data = {} for criterion in node.get('criteria', []): title = criterion.get('title') value = criterion.get('value') data_type = criterion.get('data_type') if data_type == 1: # Ordinal data type numeric_value = ordinal_value_mapping.get(value, None) if numeric_value is not None: criteria_data[title] = numeric_value elif data_type == 5: # Boolean data type boolean_value = boolean_value_mapping.get(value, None) if boolean_value is not None: criteria_data[title] = boolean_value else: # Numeric and other types try: criteria_data[title] = float(value) except ValueError: # Handle or log the error for values that can't be converted to float pass temp_data_table[node_id] = criteria_data # Collect all criteria titles criteria_titles.extend(criteria_data.keys()) # Remove duplicates from criteria titles criteria_titles = list(set(criteria_titles)) # Initialize the final data table data_table = {title: [] for title in criteria_titles} # Populate the final data table for node_id, criteria_data in temp_data_table.items(): for title, value in criteria_data.items(): data_table[title].append(value) # Format relative weight restriction data formatted_relative_wr_data = [] for relative_wr in relative_wr_data: formatted_relative_wr = { 'LHSCriterion': relative_wr.get('LHSCriterion'), 'Operator': relative_wr.get('Operator'), 'Intense': relative_wr.get('Intense'), 'RHSCriterion': relative_wr.get('RHSCriterion') } formatted_relative_wr_data.append(formatted_relative_wr) # Format immediate weight restriction data formatted_immediate_wr_data = [] for immediate_wr in immediate_wr_data: formatted_immediate_wr = { 'Criterion': immediate_wr.get('Criterion'), 'Operator': immediate_wr.get('Operator'), 'Value': immediate_wr.get('Value') } formatted_immediate_wr_data.append(formatted_immediate_wr) return data_table, formatted_relative_wr_data, formatted_immediate_wr_data, node_names, node_ids # Used to save data for each application from Frontend def save_app_data(json_data): # Extract app data and app_id app_data = json_data[0][0] # Assuming the first element contains the app_id app_id = app_data['app_id'] # Directory setup app_dir = f"app_dirs/{app_id}" if not os.path.exists(app_dir): os.makedirs(app_dir) # New data structure with additional attributes structured_data = { "app_id": app_id, "nodeNames": json_data[1], "selectedCriteria": json_data[2], "gridData": json_data[3], "relativeWRData": json_data[4], "immediateWRData": json_data[5], "results": json_data[6] } # Save the newly structured data to a JSON file # with open(os.path.join(app_dir, "data.json"), 'w', encoding='utf-8') as f: # json.dump(structured_data, f, ensure_ascii=False, indent=4) with open(os.path.join(app_dir, f"{app_id}_data.json"), 'w', encoding='utf-8') as f: json.dump(structured_data, f, ensure_ascii=False, indent=4) return app_data # Used to check if a JSON file for a given application ID exists def check_json_file_exists(app_id): app_dir = f"app_dirs/{app_id}" # The directory where the JSON files are stored file_path = os.path.join(app_dir, f"{app_id}_data.json") return os.path.exists(file_path) def create_data_table(selected_criteria, extracted_data, field_mapping): # Initialize the data table with lists for each criterion data_table = {criterion: [] for criterion in selected_criteria} # Loop over each node in the extracted data for node in extracted_data: # For each selected criterion, retrieve the corresponding value from the node's data for criterion in selected_criteria: # Determine the field name using the mapping, defaulting to the criterion name itself field_name = field_mapping.get(criterion, criterion) value = None # Default value if field is not found # Special case for hardware attributes if 'hardware' in node and field_name in node['hardware']: value = node['hardware'][field_name] elif field_name in node: value = node[field_name] # Replace zero value with 0.00001 if value == 0: # value = 0.00001 value = 10 data_table[criterion].append(value) return data_table # Used to convert RAM and # of Cores def convert_data_table(created_data_table): # Check if 'Number of CPU Cores' exists in the dictionary and convert its values if 'Number of CPU Cores' in created_data_table: created_data_table['Number of CPU Cores'] = [1/x for x in created_data_table['Number of CPU Cores']] # Check if 'Memory Size' exists in the dictionary and convert its values if 'Memory Size' in created_data_table: created_data_table['Memory Size'] = [1/x for x in created_data_table['Memory Size']] return created_data_table # Used to Append "Score" and "Rank" for each node in SAL's response JSON def append_evaluation_results(sal_reply_body, scores_and_ranks): # Check if sal_reply_body is a string and convert it to a Python object if isinstance(sal_reply_body, str): sal_reply_body = json.loads(sal_reply_body) # Check if there is only one node and scores_and_ranks are empty if len(sal_reply_body) == 1 and not scores_and_ranks: # Directly assign score and rank to the single node sal_reply_body[0]["score"] = 1 sal_reply_body[0]["rank"] = 1 return sal_reply_body # Proceed if there are multiple nodes or scores_and_ranks is not empty # Create a dictionary mapping Ids to scores and ranks eval_results_dict = {result['Id']: (result['DEA Score'], result['Rank']) for result in scores_and_ranks if scores_and_ranks} # Iterate over each node in sal_reply_body and append Score and Rank for node in sal_reply_body: node_id = node.get('id') # Assuming the ID is directly under the node if node_id in eval_results_dict: score, rank = eval_results_dict[node_id] node["score"] = score node["rank"] = rank return sal_reply_body def convert_value(value, criterion_info, is_matched): if criterion_info['type'] == 5: # Boolean type return 1 if value else 0 elif criterion_info['type'] == 1: # Ordinal type if is_matched: # For matched nodes, use the mapping ordinal_value_mapping = {"High": 3, "Medium": 2, "Low": 1} return ordinal_value_mapping.get(value, value) # Use the value from mapping, or keep it as is if not found else: # For unmatched nodes, assign default value return 1 return value # Used to read the saved application data CFSB when triggered by Optimizer def read_application_data(app_id, sal_reply_body): app_dir = os.path.join("app_dirs", app_id) file_path = os.path.join(app_dir, f"{app_id}_data.json") data_table, relative_wr_data, immediate_wr_data, node_names, node_ids = {}, [], [], [], [] if isinstance(sal_reply_body, str): sal_reply_body = json.loads(sal_reply_body) if os.path.exists(file_path): print(f"JSON file found for application ID {app_id}.") with open(file_path, 'r', encoding='utf-8') as f: data = json.load(f) selected_criteria = {criterion['title']: criterion for criterion in data.get('selectedCriteria', [])} # Define the default list criteria mapping default_list_criteria_mapping = { "Operating cost": "price", "Memory Price": "memoryPrice", "Number of CPU Cores": "cores", "Memory Size": "ram", "Storage Capacity": "disk" } for criterion in selected_criteria: data_table[criterion] = [] matched_node_ids = [node['id'] for node in data.get('gridData', []) if node['id'] in [n['id'] for n in sal_reply_body]] unmatched_node_ids = [n['id'] for n in sal_reply_body if n['id'] not in matched_node_ids] # Process MATCHED nodes for node in data.get('gridData', []): if node['id'] in matched_node_ids: node_ids.append(node['id']) # node_names.append(node.get('name', 'Unknown')) for crit, criterion_info in selected_criteria.items(): value = next((criterion['value'] for criterion in node['criteria'] if criterion['title'] == crit), None) converted_value = convert_value(value, criterion_info, is_matched=True) data_table[crit].append(converted_value) # Process UNMATCHED nodes for node_id in unmatched_node_ids: node_data = next((node for node in sal_reply_body if node['id'] == node_id), {}) node_ids.append(node_id) for criterion, crit_info in selected_criteria.items(): mapped_field = default_list_criteria_mapping.get(criterion, '') value = node_data.get(mapped_field, 0.001 if crit_info['type'] == 2 else False) converted_value = convert_value(value, crit_info, is_matched=False) data_table[criterion].append(converted_value) node_names = node_ids relative_wr_data, immediate_wr_data = data.get('relativeWRData', []), data.get('immediateWRData', []) else: # There is not any node id match - Proceed only with the nodes from SAL's reply print(f"No JSON file found for application ID {app_id}. Proceed only with data from SAL.") extracted_data_SAL, node_ids_SAL, node_names_SAL = extract_SAL_node_candidate_data(sal_reply_body) selected_criteria = ["Number of CPU Cores", "Memory Size"] field_mapping = create_criteria_mapping() data_table = create_data_table(selected_criteria, extracted_data_SAL, field_mapping) # Assign relativeWRData and immediateWRData regardless of node ID matches relative_wr_data = [] immediate_wr_data = [] node_ids = node_ids_SAL node_names = node_ids return data_table, relative_wr_data, immediate_wr_data, node_names, node_ids # Used to generate random values for DataGrid def random_value_based_on_type(data_type, criterion_info=None): if data_type == 1: # Ordinal # Assuming 'values' are part of criterion_info for ordinal types return random.choice(criterion_info.get('values', ["High", "Medium", "Low"])) elif data_type == 5: # Boolean return random.choice([True, False]) else: # Numeric # Default case for numeric types return round(random.uniform(1, 100), 2) # Used to parse Dummy JSON files for Review def read_json_file_as_string(file_path): try: with open(file_path, 'r') as file: return file.read() except Exception as e: print(f"Error reading JSON file: {e}") return None # Used to transform SAL's response before sending to DataGrid # This version is designed to read the structure of SAL's response obtained from POSTMAN def extract_node_candidate_data(json_file_path): with open(json_file_path, 'r') as file: json_data = json.load(file) extracted_data = [] node_ids = [] node_names = [] for item in json_data: hardware_info = item.get("nodeCandidate", {}).get("hardware", {}) node_data = { "name": item['name'], "id": item['id'], "nodeId": item.get("nodeCandidate", {}).get("nodeId"), "nodeCandidateType": item.get("nodeCandidate", {}).get("nodeCandidateType"), "price": item.get("nodeCandidate", {}).get("price", 0.0), "pricePerInvocation": item.get("nodeCandidate", {}).get("pricePerInvocation", 0.0), "memoryPrice": item.get("nodeCandidate", {}).get("memoryPrice", 0.0), "hardware": { "id": hardware_info.get("id"), "name": hardware_info.get("name"), "providerId": hardware_info.get("providerId"), "cores": hardware_info.get("cores"), "ram": hardware_info.get("ram") * 1024 if hardware_info.get("ram") else None, # Assuming RAM needs conversion from GB to MB "disk": hardware_info.get("disk"), "fpga": hardware_info.get("fpga") } } extracted_data.append(node_data) node_ids.append(item['id']) node_names.append(item.get('name', '')) return extracted_data, node_ids, node_names