Henrik Wahlqvist 0a8a2d69a6 Add option for generating RTE dummy files
Change-Id: Ie90031808b8cf47f6542caa73014445c803a2f94
2025-01-16 10:57:37 +01:00

513 lines
20 KiB
Python

# Copyright 2024 Volvo Car Corporation
# Licensed under Apache 2.0.
"""Performs compatibility upgrade of models to PyBuild using Matlab."""
import argparse
import hashlib
import json
import os
import sys
from pathlib import Path
from re import search
from typing import List, Optional
try:
from importlib.resources import files
except ImportError:
from importlib_resources import files
import shutil
import git
from powertrain_build import build, pt_matlab
from powertrain_build.lib import logger, helper_functions
LOGGER = logger.create_logger(__file__)
class PyBuildWrapper(pt_matlab.Matlab):
"""Performs upgrade of Matlab models to PyBuild system."""
HASH_FILE_NAME = "pybuild_file_hashes.json"
PARSER_HELP = "Run PyBuild, update and/or generate code for selected models and/or build."
def __init__(self, args):
"""Constructor, initializes paths for PyBuild upgrader.
Args:
args (argument parser): see add_args static method.
"""
super().__init__(dry_run=args.dry_run, matlab_bin=args.matlab_bin, nojvm=False)
self.root_path = helper_functions.get_repo_root()
self.repo = git.Repo(self.root_path)
self.target_link_settings_hash_file_path = Path("ConfigDocuments", "target_link_settings_file_hashes.json")
self.target_link_settings_folders = [
Path("ConfigDocuments", "targetlinkSettings"),
Path("ConfigDocuments", "TL4_3_settings"),
]
self.update = args.update
self.codegen = args.codegen
self.build = args.build
self.build_specific = getattr(args, "project_config", None) is not None
self.project_config = self._set_project_configuration(args)
self.generate_system_info = getattr(args, "generate_system_info", False)
self.core_dummy = getattr(args, "core_dummy", True)
self.rte_dummy = getattr(args, "rte_dummy", False)
self.debug = getattr(args, "debug", True)
self.no_abort = getattr(args, "no_abort", True)
self.no_nvm_a2l = getattr(args, "no_nvm_a2l", False)
self.complete_a2l = getattr(args, "complete_a2l", False)
self.silver_a2l = getattr(args, "silver_a2l", False)
self.generate_rte_checkpoint_calls = getattr(args, "generate_rte_checkpoint_calls", False)
self.interface = args.interface
self.matlab_include = args.include
# Always default to conversion table in pytools/config/conversion_table.json.
# Unless otherwise specified.
conv_tab_path = getattr(args, "generate_custom_conv_tab", None)
default_conv_tab_path = os.path.join("ConfigDocuments", "pytools_settings", "conversion_table.json")
self.conv_tab = conv_tab_path if conv_tab_path is not None else default_conv_tab_path
self.should_run, self.models = self._evaluate_run_and_models(args)
@staticmethod
def _set_project_configuration(args):
"""Evaluate path to project configuration file.
Args:
args (argument parser): see add_args static method.
Returns:
project_config: Path to project configuration file.
"""
if getattr(args, "project_config", None) is not None:
project_config = args.project_config.replace("/", os.sep)
elif args.build is not None:
if args.build.lower() == "custom":
# TODO: Change other scripts to accept custom config
# Change this to point to a config file instead
project_config = os.path.join(os.environ.get("PROJECT_DIR"), "ProjectCfg.json")
else:
project_config = os.path.join("Projects", args.build, "ProjectCfg.json")
else:
project_config = None
return project_config
@staticmethod
def convert_path_sep(paths):
"""Matlab requires forward-spaces for model paths. Convert backslashes.
Args:
paths (list): Model paths to fix path separators for.
Returns:
(list): Models paths separated with forward slashes.
"""
return [path.replace("\\", "/") for path in paths]
@staticmethod
def get_matlab_scripts_commit_sha():
"""Get current commit sha of matlab-scripts submodule, if available.
Returns:
matlab_scripts_commit_sha (str): Commit sha of matlab-scripts submodule.
None, if not available.
"""
repo = git.Repo()
try:
matlab_scripts = repo.submodule("matlab-scripts")
matlab_scripts_commit_sha = matlab_scripts.hexsha
except ValueError:
LOGGER.info("Submodule matlab-scripts not available, " "skipping adding its commit sha to hash file.")
matlab_scripts_commit_sha = None
return matlab_scripts_commit_sha
@staticmethod
def read_bytes(file_path):
"""Read file contents in byte mode.
Args:
file_path (Path): Path to file.
Returns:
file_bytes (bytes): Content of file read in binary mode.
"""
if file_path.suffix in [".slx", ".mdl", ".mexw64"]:
# Already treated as binary file
file_bytes = file_path.read_bytes()
else:
# Hack to make this script os independent
with file_path.open(encoding="iso-8859-1") as file_handle:
content = file_handle.read()
file_bytes = content.encode("iso-8859-1")
return file_bytes
@staticmethod
def get_files_to_hash(model_folder):
"""Get files to generate hashes for (PyBuild specific files).
Args:
model_folder (Path): Path to model folder.
Returns:
files_to_hash (list): List of, pybuild specific, files to generate hashes for.
"""
valid_file_endings = [
# source files
".a2l",
".c",
".h",
".mexw64",
".tlc",
# config files
".json",
# model files
".mdl",
".m",
]
valid_folders = [
model_folder,
Path(model_folder, "matlab_src"),
Path(model_folder, "pybuild_cfg"),
Path(model_folder, "pybuild_src"),
]
hash_file_path = Path(model_folder, PyBuildWrapper.HASH_FILE_NAME)
model_files = list(model_folder.rglob("*.*"))
if hash_file_path.exists():
model_files.remove(hash_file_path)
files_to_hash = [f for f in model_files if f.parent in valid_folders and f.suffix in valid_file_endings]
return files_to_hash
@staticmethod
def get_shared_function_files():
"""Get shared function files, files generated by VcSharedFunctions.mdl.
Returns:
shared_function_files (list): List of files generated by VcSharedFunctions.mdl.
"""
shared_function_files = []
common_source_folder = Path("Models/Common/pybuild_src")
for source_file in common_source_folder.glob("*.*"):
with source_file.open() as file_handle:
content = file_handle.read()
if search(r"Simulink model\s+: VcSharedFunctions", content) is not None:
shared_function_files.append(source_file)
return shared_function_files
@staticmethod
def get_file_hashes(model_paths, write_to_file=True):
"""Calculate SHA256 file hashes for files in given model folders.
Args:
model_paths (list): List of model paths.
write_to_file (bool): True/False whether calculated hashes should be written to file.
Returns:
model_to_files_hash_dict (dict): Dict mapping model name to files and its hashes.
"""
model_to_files_hash_dict = {}
model_folders = [(Path(m).parent, Path(m).stem) for m in model_paths]
for model_folder, model_name in model_folders:
file_to_hash_dict = {}
files_to_hash = PyBuildWrapper.get_files_to_hash(model_folder)
if "VcSharedFunctions" in model_folder.as_posix():
shared_function_files = PyBuildWrapper.get_shared_function_files()
files_to_hash.extend(shared_function_files)
for file_to_hash in files_to_hash:
file_bytes = PyBuildWrapper.read_bytes(file_to_hash)
file_to_hash_dict[file_to_hash.name] = hashlib.sha256(file_bytes).hexdigest()
commit_sha = PyBuildWrapper.get_matlab_scripts_commit_sha()
if commit_sha is not None:
file_to_hash_dict["matlab-scripts"] = commit_sha
if write_to_file:
with Path(model_folder, PyBuildWrapper.HASH_FILE_NAME).open("w", encoding="utf-8") as file_handle:
json.dump(file_to_hash_dict, file_handle, indent=4)
model_to_files_hash_dict[model_name] = file_to_hash_dict
return model_to_files_hash_dict
def _evaluate_run_and_models(self, args):
"""Evaluate if PyBuild Matlab related parts should run.
Additionally, it sets which models to update and/or generate code for, based on arguments.
NOTE: model_list=None indicates all models when run is True. Deprecated, add force flag?
Args:
args (argument parser): see add_args static method.
Returns:
should_run: True/False whether PyBuild should run or not.
model_list: List of models to update/generate code for.
"""
run_powertrain_build = args.build is not None or getattr(args, "project_config", None) is not None
should_run = args.update or args.codegen or run_powertrain_build
model_list = args.models if args.models else self.get_changed_models()
LOGGER.info("Affected models: %s", ", ".join(model_list))
if not model_list:
# PyBuild should not run if there were no model changes
should_run = run_powertrain_build
model_list = None
return should_run, model_list
def regenerate_target_link_settings_file_hashes(self):
"""Regenerate the file mapping TargetLink settings files to their hashes."""
target_link_settings_file_hashes = self.calculate_target_link_settings_file_hashes()
with self.target_link_settings_hash_file_path.open("w", encoding="utf-8") as file_handle:
json.dump(target_link_settings_file_hashes, file_handle, indent=4)
def calculate_target_link_settings_file_hashes(self):
"""Calculate SHA256 file hashes for files in TargetLink settings folders.
Returns:
file_to_hash_dict (dict): Dict mapping settings files and their hashes.
"""
file_to_hash_dict = {}
for settings_folder in self.target_link_settings_folders:
for settings_file in settings_folder.rglob("*.*"):
file_bytes = self.read_bytes(settings_file)
file_to_hash_dict[settings_file.name] = hashlib.sha256(file_bytes).hexdigest()
return file_to_hash_dict
def verify_target_link_settings(self):
"""Verify current TargetLink settings, comparing against commited file hashes file.
Returns:
(bool): True/False, depending on if the TargetLink settings have changed.
"""
if not self.target_link_settings_hash_file_path.exists():
message = (
"Could not read TargetLink settings file hashes file: "
f"{self.target_link_settings_hash_file_path.as_posix()}.\n"
"If your repo runs the jobb PyBuildDiff in hash mode, make sure to generate one."
"Ignoring settings verification."
)
LOGGER.warning(message)
return True
with self.target_link_settings_hash_file_path.open(encoding="utf-8") as file_handle:
current_file_hashes_dict = json.load(file_handle)
new_file_hashes_dict = self.calculate_target_link_settings_file_hashes()
return new_file_hashes_dict == current_file_hashes_dict
def get_changed_models(self):
"""Get changed models in current commit."""
changed_files_tmp = self.repo.git.diff("--diff-filter=d", "--name-only", "HEAD~1")
changed_files = changed_files_tmp.splitlines()
changed_models = [m for m in changed_files if m.endswith(".mdl") or m.endswith(".slx")]
return changed_models
def check_generate_shared_functions(self, matlab_command_list):
"""Check if shared function files should be generated.
Args:
matlab_command_list ([str]): list of matlab commands.
"""
if [model for model in self.models if model.endswith("VcSharedFunctions.mdl")]:
matlab_command_list.append(pt_matlab.cmd_callfunc("generateSharedFunctions", True))
def build_automation(self, mode):
"""Run Matlab with a specific task and specific models.
Args:
mode (str): Matlab run mode (update, codegen).
Returns:
exit_code (int): Exit code from Matlab.
"""
# Will be used in submodule matlab-scripts (if up to date), CodeGen/updateCodeSwConfig.m
calling_python = sys.version_info
calling_python_string = f"py -{calling_python.major}.{calling_python.minor}"
os.environ.setdefault("CALLING_PYTHON", calling_python_string)
# Specify a new script and log file name for each run
script_name = f"powertrain_build_matlab_{mode}.m"
self.log = f"powertrain_build_matlab_{mode}.log"
# Set up matlab path
matlab_scripts_path = os.path.join("powertrain_build_matlab_scripts")
# Copy matlab-scripts to the project root
matlab_script_folder = files("powertrain_build.matlab_scripts")
shutil.rmtree(matlab_scripts_path, ignore_errors=True)
shutil.copytree(matlab_script_folder, matlab_scripts_path)
cmds = []
cmds.append(f"cd '{self.root_path}'")
cmds.append(pt_matlab.cmd_path(matlab_scripts_path, True))
cmds.append(pt_matlab.cmd_path(self.matlab_include, True))
if mode == "codegen":
cmds.append(pt_matlab.cmd_callfunc("loadLibraries", self.matlab_include))
# Generate the command for calling the main build script:
args = [mode, True]
if self.models:
self.check_generate_shared_functions(cmds)
args.append(self.convert_path_sep(self.models))
cmds.append(pt_matlab.cmd_callfunc("BuildAutomationPyBuild", *args))
pt_matlab.write_m_script(script_name, pt_matlab.cmds_join(cmds), wrap_cmd=False)
# Reset the Matlab watcher before running a new m-script
self.matlab_watcher.reset_errors()
self.run_m_script(script_name, wrap_cmd=False, attempts=2)
if not self.matlab_watcher.task_success:
LOGGER.error("PyBuild %s error!", mode)
return 1
if not self.verify_target_link_settings():
LOGGER.error("Your TargetLink settings differ from the ones set by current branch.")
return 1
self.get_file_hashes(self.models)
return 0
def run_powertrain_build(self):
"""Execute powertrain-build.
Returns:
exit_code: Exit code from powertrain-build build step.
"""
try:
exit_code = build.build(
self.project_config,
interface=self.interface,
core_dummy=self.core_dummy,
rte_dummy=self.rte_dummy,
no_abort=self.no_abort,
no_nvm_a2l=self.no_nvm_a2l,
debug=self.debug,
generate_system_info=self.generate_system_info,
generate_custom_conversion_table=self.conv_tab,
complete_a2l=self.complete_a2l,
silver_a2l=self.silver_a2l,
generate_rte_checkpoint_calls=self.generate_rte_checkpoint_calls,
)
except (FileNotFoundError, PermissionError) as ex:
LOGGER.error(ex)
exit_code = 1
return exit_code
def run(self):
"""Run PyBuild, update and/or generate code for selected models and/or build.
Returns:
exit_code: Exit code from Matlab and build step.
"""
if not self.should_run:
return 0
LOGGER.info("Preparing workspace for PyBuild!")
exit_code = 0
if self.update:
LOGGER.info("Running PyBuild update!")
exit_code |= self.build_automation(mode="update")
if self.codegen:
LOGGER.info("Running PyBuild generate code!")
exit_code |= self.build_automation(mode="codegen")
if self.build:
LOGGER.info("Running PyBuild.build for %s!", self.build)
exit_code |= self.run_powertrain_build()
if self.build_specific:
LOGGER.info("Running PyBuild specific build for %s!", self.project_config)
exit_code |= self.run_powertrain_build()
return exit_code
@staticmethod
def add_args(parser):
"""Add expected arguments to the supplied parser.
Args:
parser (ArgumentParser): parser to add arguments to.
"""
parser.add_argument(
"--dry-run",
action="store_true",
help="Dry run: No changes, simulation only (default: False).",
)
parser.add_argument("--repo-root", help="Path to repository where work should be done")
parser.add_argument(
"--interface",
action="store_true",
help="Create interface consistency report (default: False)" "NOTE: This requires the --build flag.",
)
parser.add_argument(
"--models",
nargs="+",
default=None,
help="List of model files (full path, "
"e.g. Models/<SSP>/<MODEL>/<MDL-FILE>) to upgrade, separated with "
"spaces. Takes precedence over Git HEAD.",
)
parser.add_argument(
"--update",
action="store_true",
help="Run PyBuild update on models (default: False).",
)
parser.add_argument(
"--codegen",
action="store_true",
help="Run PyBuild code generation on models (default: False). "
"NOTE: This requires either the --update flag or already updated models.",
)
parser.add_argument(
"--build",
default=None,
help="Run PyBuild for project with standard settings, VCC SPM SW release " "(default: None).",
)
parser.add_argument(
"--regenerate-tl-settings-hashes",
action="store_true",
help="Regenerate the file mapping TargetLink settings files to their hashes.",
)
parser.add_argument(
"--include",
default="matlab-scripts",
help="Path to folder containing Matlab scripts and simulink libraries to include.",
)
parser.set_defaults(func=run_wrapper)
subparsers = parser.add_subparsers(help="PyBuild specific build.")
build_specific_parser = subparsers.add_parser(
"build-specific", help="Run PyBuild for project with specific settings."
)
build.add_args(build_specific_parser)
# Matlab arguments added by parent:
pt_matlab.Matlab.add_args(parser)
def run_wrapper(args: argparse.Namespace) -> int:
"""Run PyBuildWrapper."""
if args.build is not None and getattr(args, "project_config", None) is not None:
LOGGER.error("Cannot run both PyBuild quick build (--build <PROJECT>) " "and specific build (build-specific).")
return 1
wrapper = PyBuildWrapper(args)
if args.regenerate_tl_settings_hashes:
wrapper.regenerate_target_link_settings_file_hashes()
return 0
return wrapper.run()
def main(argv: Optional[List[str]] = None) -> int:
"""Run PyBuildWrapper"""
parser = argparse.ArgumentParser("PyBuild Wrapper")
PyBuildWrapper.add_args(parser)
args = parser.parse_args(argv)
return run_wrapper(args)
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))