Fixes for ZC diagnostics

* yaml2arxml needs all entries from the diagnostics yaml file.
* DIDs > 1 bytes need to be converted to arrays

Change-Id: I2087c1733131d3b46d028bd49d4c0bb7dc16fbbd

pybuild/core.py

@@ -337,12 +337,12 @@ class ZCCore(ProblemLogger):
         Returns:
             (dict): Dict of DTCs, project yaml dict where the keys also appear in the project.
         """
-        yaml_dtcs = set(event_data.keys())
+        valid_dtcs = {}
-        dtcs_not_in_yaml = self.project_dtcs - yaml_dtcs
+
+        dtcs_not_in_yaml = self.project_dtcs - set(event_data.keys())
         for key in dtcs_not_in_yaml:
             self.warning(f'Ignoring DTC {key} since it does not appear in the project diagnostics yaml file.')
 
-        valid_dtcs = {}
         supported_operations = {"SetEventStatus"}
         for dtc_name, dtc_data in event_data.items():
             if dtc_name not in self.project_dtcs:

pybuild/dids.py

@@ -15,7 +15,7 @@ from ruamel.yaml import YAML
 from pybuild import build_defs
 from pybuild.lib.helper_functions import deep_dict_update
 from pybuild.problem_logger import ProblemLogger
-from pybuild.types import get_ec_type, get_float32_types
+from pybuild.types import byte_size, get_ec_type, get_float32_types
 from pybuild.unit_configs import CodeGenerators
 
 
@@ -637,28 +637,19 @@ class ZCDIDs(ProblemLogger):
         """
         self._valid_dids = {}
 
-        yaml_did_names = set(yaml_dids.keys())
+        dids_not_in_yaml = set(self.project_dids.keys()) - set(yaml_dids.keys())
-        dids_not_in_project = yaml_did_names - set(self.project_dids.keys())
+        for did in dids_not_in_yaml:
-        for did in dids_not_in_project:
+            self.warning(f'DID {did} not defined in project diagnostics yaml file.')
-            self.warning(f'Ignoring DID {did}, not defined in any model.')
 
-        supported_operations = set(self.get_operation_data().keys())
+        for did, did_data in yaml_dids.items():
-        for did, did_data in self.project_dids.items():
+            if did not in self.project_dids:
-            if did not in yaml_dids:
+                self.warning(f'Ignoring DID {did}, not defined in any model.')
-                self.warning(f'DID {did} not defined in project diagnostics yaml file.')
                 continue
-
+            data_type = self.project_dids[did]['type']
-            yaml_operations = set(yaml_dids[did]['operations'].keys())
+            if not data_type.startswith('UInt'):
-            if not yaml_operations.issubset(supported_operations):
+                self.warning(f'Ignoring DID {did} of type {data_type}, only unsigned integers are supported.')
-                self.warning(f'Ignoring unsupported operations {yaml_operations - supported_operations} for DID {did}.')
+                continue
-
+            self._valid_dids[did] = did_data
-            operations = {
-                "operations": {
-                    operation: {} for operation in supported_operations
-                }
-            }
-
-            self._valid_dids[did] = deep_dict_update(did_data, operations)
 
     def _get_project_dids(self):
         """Return a dict with DIDs defined in the project.
@@ -673,27 +664,42 @@ class ZCDIDs(ProblemLogger):
             return {}
         return project_dids
 
-    def get_operation_data(self, operation=None):
+    def get_operation_data(self, operation, did_data):
         """Get operation function data of supported operations.
 
         Args:
             operation (str): Operation to get data for.
+            did_data (dict): DID data.
         Returns:
             (dict): Operation function data.
         """
+        array_size = byte_size(did_data['type'])
+        if array_size > 1:
+            read_data_declaration = f'UInt8 Run_{did_data["name"]}_ReadData(UInt8 Data[{array_size}])'
+            read_data_definition = (
+                '{\n'
+                f'  for (UInt8 i = 0U; i < {array_size}; i++) {{\n'
+                f'    Data[i] = ({did_data["name"]} >> (8 * i)) & 0xFF;\n'
+                '  }\n'
+                '  return 0U;\n'
+                '}\n'
+            )
+        else:
+            read_data_declaration = f'UInt8 Run_{did_data["name"]}_ReadData(UInt8 *Data)'
+            read_data_definition = (
+                '{\n'
+                f'  *Data = {did_data["name"]};\n'
+                '  return 0U;\n'
+                '}\n'
+            )
         operation_data = {
             'ReadData': {
-                'declaration': 'UInt8 Run_{did_name}_ReadData({data_type} *Data)',
+                'declaration': read_data_declaration,
-                'body': (
+                'body': read_data_definition,
-                    '{{\n'
-                    '  *Data = {did_name};\n'
-                    '  return 0U;\n'
-                    '}}\n'
-                ),
             }
         }
-        if operation is None:
+        if operation not in operation_data:
-            return operation_data
+            return None
         return operation_data[operation]
 
     def _get_header_file_content(self):
@@ -717,23 +723,30 @@ class ZCDIDs(ProblemLogger):
         if not self.valid_dids:
             return header + footer
 
-        body = [f'#include "{build_defs.PREDECL_DISP_ASIL_D_START}"\n']
+        variable_declarations = []
-        for did_data in self.valid_dids.values():
+        function_declarations = []
-            define = did_data["class"].split('/')[-1]  # E.q. for ASIL D it is ASIL_D/CVC_DISP_ASIL_D
+        for did, did_data in self.valid_dids.items():
-            body.append(f'extern {define} {did_data["type"]} {did_data["name"]};\n')
+            define = self.project_dids[did]["class"].split('/')[-1]  # E.q. for ASIL D it is ASIL_D/CVC_DISP_ASIL_D
-        body.append(f'#include "{build_defs.PREDECL_DISP_ASIL_D_END}"\n')
+            variable_declarations.append(
-
+                f'extern {define} {self.project_dids[did]["type"]} {self.project_dids[did]["name"]};\n'
-        body.append(f'\n#include "{build_defs.PREDECL_CODE_ASIL_D_START}"\n')
-        for did_data in self.valid_dids.values():
-            body.extend(
-                [
-                    self.get_operation_data(operation)['declaration'].format(
-                        did_name=did_data["name"],
-                        data_type=did_data["type"]
-                    ) + ';\n' for operation in did_data["operations"]
-                ]
             )
-        body.append(f'#include "{build_defs.PREDECL_CODE_ASIL_D_END}"\n')
+            for operation in did_data["operations"]:
+                operation_data = self.get_operation_data(operation, self.project_dids[did])
+                if operation_data is None:
+                    self.warning(
+                        f'Will not generate code for unsupported operation {operation}. Add manually for DID {did}.'
+                    )
+                    continue
+                function_declarations.append(operation_data['declaration'] + ';\n')
+
+        body = [
+            f'#include "{build_defs.PREDECL_DISP_ASIL_D_START}"\n',
+            *variable_declarations,
+            f'#include "{build_defs.PREDECL_DISP_ASIL_D_END}"\n',
+            f'\n#include "{build_defs.PREDECL_CODE_ASIL_D_START}"\n',
+            *function_declarations,
+            f'#include "{build_defs.PREDECL_CODE_ASIL_D_END}"\n',
+        ]
 
         return header + body + footer
 
@@ -752,14 +765,12 @@ class ZCDIDs(ProblemLogger):
             return header
 
         body = [f'#include "{build_defs.CVC_CODE_ASIL_D_START}"\n']
-        for did_data in self.valid_dids.values():
+        for did, did_data in self.valid_dids.items():
             for operation in did_data["operations"]:
-                body.append(
+                operation_data = self.get_operation_data(operation, self.project_dids[did])
-                    self.get_operation_data(operation)['declaration'].format(
+                if operation_data is None:
-                        did_name=did_data["name"],
+                    continue  # Warning already given in header generation
-                        data_type=did_data["type"]
+                body.append(operation_data['declaration'] + '\n' + operation_data['body'])
-                    ) + '\n' + self.get_operation_data(operation)['body'].format(did_name=did_data["name"])
-                )
         body.append(f'#include "{build_defs.CVC_CODE_ASIL_D_END}"\n')
 
         return header + body

pybuild/zone_controller/composition_yaml.py

@@ -298,46 +298,27 @@ class CompositionYaml(ProblemLogger):
 
         return trigger_signal
 
-    def _get_diagnostic_event_info(self, event_dict):
-        """Get diagnostic event information from an even dictionary.
-
-        Args:
-            event_dict (dict): Dict with event information.
-        Returns:
-            valid_event_dict (dict): Dict with diagnostic event information supported by yaml2arxml script.
-        """
-        valid_event_dict = {}
-        dtcs = self.zc_core.get_diagnostic_trouble_codes(event_dict)
-        for dtc_name, dtc_data in dtcs.items():
-            valid_event_dict[dtc_name] = {"operations": dtc_data["operations"], "runnable": dtc_data["runnable"]}
-        return valid_event_dict
-
-    def _get_diagnostic_did_info(self, did_dict):
-        """Get diagnostic DID information from a DID dictionary.
-        NOTE: This function sets the valid_dids property of the ZCDIDs object.
-
-        Args:
-            did_dict (dict): Dict with DID information.
-        Returns:
-            valid_did_dict (dict): Dict with diagnostic DID information supported by yaml2arxml script.
-        """
-        valid_did_dict = {}
-        self.zc_dids.valid_dids = did_dict
-        for did_name, did_data in self.zc_dids.valid_dids.items():
-            valid_did_dict[did_name] = {"operations": did_data["operations"]}
-        return valid_did_dict
-
     def _get_diagnostic_info(self):
         """Get diagnostic information from composition spec.
+        NOTE: This function sets the valid_dids property of the ZCDIDs object.
 
         Returns:
             (dict): Dict containing diagnostic information.
         """
-        diag_dict = self.composition_spec.get("Diagnostics", {})
+        diag_dict = {}
-        return {
+        diagnostics = self.composition_spec.get("Diagnostics", {})
-            "events": self._get_diagnostic_event_info(diag_dict.get("events", {})),
+        dids = diagnostics.get("dids", {})
-            "dids": self._get_diagnostic_did_info(diag_dict.get("dids", {})),
+        events = diagnostics.get("events", {})
-        }
+        rids = diagnostics.get("rids", {})
+        if dids:
+            self.zc_dids.valid_dids = dids
+            diag_dict["dids"] = self.zc_dids.valid_dids
+        if events:
+            diag_dict["events"] = self.zc_core.get_diagnostic_trouble_codes(events)
+        if rids:
+            diag_dict["rids"] = rids
+            self.warning('Will not generate code for RIDs, add manually.')
+        return diag_dict
 
     def _get_ports_info(self):
         """Creates a dict containing port information.

test_data/pybuild/test_dids/zc_dids.py

@@ -32,6 +32,35 @@ dummy_project_dids = {
     }
 }
 
+bad_dummy_project_dids = {
+    'dummy_did_one': {
+        'handle': 'VcDummy/VcDummy/Subsystem/VcDummy/VcDummy/1_VcDummy/Rel',
+        'name': 'dummy_did_one',
+        'configs': '((ALWAYS_ACTIVE))',
+        'description': 'Dummy DID',
+        'type': 'Float32',
+        'unit': '',
+        'offset': '',
+        'lsb': '',
+        'min': '-',
+        'max': '-',
+        'class': 'ASIL_D/CVC_DISP_ASIL_D'
+    },
+    'dummy_did_two': {
+        'handle': 'VcDummy/VcDummy/Subsystem/VcDummy/VcDummy/1_VcDummyTwo/Rel',
+        'name': 'dummy_did_two',
+        'configs': '((ALWAYS_ACTIVE))',
+        'description': 'Dummy DID',
+        'type': 'Int8',
+        'unit': '',
+        'offset': '',
+        'lsb': '',
+        'min': '-',
+        'max': '-',
+        'class': 'ASIL_D/CVC_DISP_ASIL_D'
+    },
+}
+
 valid_dids = {
     'dummy_did_one': {
         'operations': {
@@ -47,6 +76,9 @@ valid_dids = {
 
 bad_valid_dids = {
     'dummy_did_one': {
+        'identifier': 0xAAAA,
+        'numberOfParameters': 2,
+        'totalNumberOfBytes': 2,
         'operations': {
             'ReadData': {'random_data': {}},
             'WriteData': {'random_data': {}},
@@ -55,6 +87,9 @@ bad_valid_dids = {
         },
     },
     'dummy_did_two': {
+        'identifier': 0xAAAB,
+        'numberOfParameters': 2,
+        'totalNumberOfBytes': 2,
         'operations': {
             'ReadData': {'random_data': {}},
             'WriteData': {'random_data': {}},
@@ -63,6 +98,9 @@ bad_valid_dids = {
         },
     },
     'dummy_did_three': {
+        'identifier': 0xAAAC,
+        'numberOfParameters': 2,
+        'totalNumberOfBytes': 2,
         'operations': {
             'ReadData': {'random_data': {}},
             'WriteData': {'random_data': {}},
@@ -74,51 +112,67 @@ bad_valid_dids = {
 
 test_valid_dids_setter_expected = {
     'dummy_did_one': {
-        'handle': 'VcDummy/VcDummy/Subsystem/VcDummy/VcDummy/1_VcDummy/Rel',
+        'identifier': 0xAAAA,
-        'name': 'dummy_did_one',
+        'numberOfParameters': 2,
-        'configs': '((ALWAYS_ACTIVE))',
+        'totalNumberOfBytes': 2,
-        'description': 'Dummy DID',
-        'type': 'UInt8',
-        'unit': '',
-        'offset': '',
-        'lsb': '',
-        'min': '-',
-        'max': '-',
-        'class': 'ASIL_D/CVC_DISP_ASIL_D',
         'operations': {
-            'ReadData': {},
+            'ReadData': {'random_data': {}},
+            'WriteData': {'random_data': {}},
+            'ShortTermAdjustment': {'random_data': {}},
+            'ReturnControlToECU': {'random_data': {}},
         },
     },
     'dummy_did_two': {
-        'handle': 'VcDummyTwo/VcDummyTwo/Subsystem/VcDummyTwo/VcDummyTwo/1_VcDummyTwo/Rel',
+        'identifier': 0xAAAB,
-        'name': 'dummy_did_two',
+        'numberOfParameters': 2,
-        'configs': '((ALWAYS_ACTIVE))',
+        'totalNumberOfBytes': 2,
-        'description': 'Dummy DID number 2',
-        'type': 'UInt8',
-        'unit': '',
-        'offset': '',
-        'lsb': '',
-        'min': '-',
-        'max': '-',
-        'class': 'ASIL_D/CVC_DISP_ASIL_D',
         'operations': {
-            'ReadData': {},
+            'ReadData': {'random_data': {}},
+            'WriteData': {'random_data': {}},
+            'ShortTermAdjustment': {'random_data': {}},
+            'ReturnControlToECU': {'random_data': {}},
         },
     },
 }
 
-test_get_operation_data_expected = {
+test_get_operation_data_did_data = {
-    'ReadData': {
+    'dummy_did_one': {
-        'declaration': 'UInt8 Run_{did_name}_ReadData({data_type} *Data)',
+        'name': 'dummy_did_one',
-        'body': (
+        'type': 'UInt8',
-            '{{\n'
+    },
-            '  *Data = {did_name};\n'
+    'dummy_did_two': {
-            '  return 0U;\n'
+        'name': 'dummy_did_two',
-            '}}\n'
+        'type': 'UInt32',
-        ),
     }
 }
 
+test_get_operation_data_expected = {
+    'dummy_did_one': {
+        'ReadData': {
+            'declaration': 'UInt8 Run_dummy_did_one_ReadData(UInt8 *Data)',
+            'body': (
+                '{\n'
+                '  *Data = dummy_did_one;\n'
+                '  return 0U;\n'
+                '}\n'
+            ),
+        },
+    },
+    'dummy_did_two': {
+        'ReadData': {
+            'declaration': 'UInt8 Run_dummy_did_two_ReadData(UInt8 Data[4])',
+            'body': (
+                '{\n'
+                '  for (UInt8 i = 0U; i < 4; i++) {\n'
+                '    Data[i] = (dummy_did_two >> (8 * i)) & 0xFF;\n'
+                '  }\n'
+                '  return 0U;\n'
+                '}\n'
+            ),
+        },
+    },
+}
+
 TEST_GET_HEADER_FILE_CONTENT_EXPECTED = (
     '#ifndef VCDIDAPI_H\n'
     '#define VCDIDAPI_H\n'

test_data/zone_controller/test_composition_yaml/composition_yaml.py

@@ -26,7 +26,7 @@ expected_result = {
                     "accesses": composition_yaml_setup.base_accesses
                 }
             },
-            "diagnostics": {"events": {}, "dids": {}},
+            "diagnostics": {},
             "static": composition_yaml_setup.base_static,
             "shared":  composition_yaml_setup.base_shared,
             "ports": {

test_data/zone_controller/test_composition_yaml/composition_yaml_with_a2l_axis_data.py

@@ -188,7 +188,7 @@ expected_result = {
                     ]
                 }
             },
-            "diagnostics": {"events": {}, "dids": {}},
+            "diagnostics": {},
             "static": composition_yaml_setup.base_static,
             "shared": {
                 "ctestName_SC_TriggerReadRteCData": {

test_data/zone_controller/test_composition_yaml/composition_yaml_with_calls_all_fields.py

@@ -32,7 +32,7 @@ expected_result = {
                     "accesses": composition_yaml_setup.base_accesses
                 }
             },
-            "diagnostics": {"events": {}, "dids": {}},
+            "diagnostics": {},
             "static": composition_yaml_setup.base_static,
             "shared":  composition_yaml_setup.base_shared,
             "ports": {

test_data/zone_controller/test_composition_yaml/composition_yaml_with_calls_no_optional_fields.py

@@ -31,7 +31,7 @@ expected_result = {
                     "accesses": composition_yaml_setup.base_accesses
                 }
             },
-            "diagnostics": {"events": {}, "dids": {}},
+            "diagnostics": {},
             "static": composition_yaml_setup.base_static,
             "shared":  composition_yaml_setup.base_shared,
             "ports": {

test_data/zone_controller/test_composition_yaml/composition_yaml_with_dids.py

@@ -59,11 +59,32 @@ expected_result = {
                 }
             },
             "diagnostics": {
-                "events": {},
                 "dids": {
                     "DID1": {
+                        "identifier": "34EE",
+                        "numberOfParameters": 1,
+                        "totalNumberOfBytes": 2,
                         "operations": {
-                            "ReadData": {},
+                            "ReadData": {
+                                "securityAccess": "",
+                                "readLocalVariables": ["MyIRV"],
+                                "writtenLocalVariables": ["MyIRV"],
+                            },
+                            "WriteData": {
+                                "securityAccess": "",
+                                "readLocalVariables": ["MyIRV3"],
+                                "writtenLocalVariables": ["MyIRV3"],
+                            },
+                            "ShortTermAdjustment": {
+                                "securityAccess": "",
+                                "readLocalVariables": ["MyIRV4"],
+                                "writtenLocalVariables": ["MyIRV4"],
+                            },
+                            "ReturnControlToECU": {
+                                "securityAccess": "",
+                                "readLocalVariables": ["MyIRV5"],
+                                "writtenLocalVariables": ["MyIRV6"],
+                            },
                         },
                     },
                 }

test_data/zone_controller/test_composition_yaml/composition_yaml_with_dtcs.py

@@ -54,9 +54,22 @@ expected_result = {
                     "DTC1": {
                         "operations": ["SetEventStatus"],
                         "runnable": ["dummy"],
+                        "identifier": "34EA",
+                        "ThresholdUnconfirmed": 1,
+                        "StepDown": 0,
+                        "JumpDown": True,
+                        "JumpDownInit": 1,
+                        "StepUp": 1,
+                        "JumpUp": True,
+                        "JumpUpInit": 127,
+                        "TestFailedLimit": 0,
+                        "TestPassedLimit": 0,
+                        "AgedDTCLimit": 0,
+                        "ConfirmedDTCLimit": 1,
+                        "DTCEventPriority": 1,
+                        "OperationCycle":  "Battery",
                     },
                 },
-                "dids": {}
             },
             "static": composition_yaml_setup.base_static,
             "shared":  composition_yaml_setup.base_shared,

tests/pybuild/test_dids.py

@@ -12,9 +12,11 @@ from pybuild.build_proj_config import BuildProjConfig
 from pybuild.dids import DIDs, HIDIDs, ZCDIDs
 from test_data.pybuild.test_dids.zc_dids import (
     dummy_project_dids,
+    bad_dummy_project_dids,
     valid_dids,
     bad_valid_dids,
     test_valid_dids_setter_expected,
+    test_get_operation_data_did_data,
     test_get_operation_data_expected,
     TEST_GET_HEADER_FILE_CONTENT_EXPECTED,
     TEST_GET_SOURCE_FILE_CONTENT_EXPECTED,
@@ -990,16 +992,33 @@ class TestZCDIDs(unittest.TestCase):
 
     def test_valid_dids_setter(self):
         """Test setting property ZCDIDs.valid_dids."""
-        self.zc_dids.valid_dids = {}
-        self.assertDictEqual(self.zc_dids.valid_dids, {})
-
         self.zc_dids.valid_dids = bad_valid_dids
         self.assertDictEqual(self.zc_dids.valid_dids, test_valid_dids_setter_expected)
 
+    def test_valid_dids_setter_none(self):
+        """Test setting property ZCDIDs.valid_dids with no dids."""
+        self.zc_dids.valid_dids = {}
+        self.assertDictEqual(self.zc_dids.valid_dids, {})
+
+    def test_valid_dids_setter_bad_project_dids(self):
+        """Test setting property ZCDIDs.valid_dids with project DIDs of wrong data type."""
+        self.zc_dids.project_dids = bad_dummy_project_dids
+        self.zc_dids.valid_dids = valid_dids
+        self.assertDictEqual(self.zc_dids.valid_dids, {})
+
+    def test_get_operation_data_none(self):
+        """Test ZCDIDs.get_operation_data with unsupported operation."""
+        self.assertIsNone(self.zc_dids.get_operation_data('Dummy', test_get_operation_data_did_data['dummy_did_one']))
+
     def test_get_operation_data(self):
         """Test ZCDIDs.get_operation_data."""
-        result = self.zc_dids.get_operation_data()
+        result = self.zc_dids.get_operation_data('ReadData', test_get_operation_data_did_data['dummy_did_one'])
-        self.assertDictEqual(result, test_get_operation_data_expected)
+        self.assertDictEqual(result, test_get_operation_data_expected['dummy_did_one']['ReadData'])
+
+    def test_get_operation_data_array(self):
+        """Test ZCDIDs.get_operation_data with DID byte size bigger than one."""
+        result = self.zc_dids.get_operation_data('ReadData', test_get_operation_data_did_data['dummy_did_two'])
+        self.assertDictEqual(result, test_get_operation_data_expected['dummy_did_two']['ReadData'])
 
     def test_get_header_file_content_no_dids(self):
         """Test ZCDIDs._get_header_file_content without DIDs."""

tests/zone_controller/test_composition_yaml.py

@@ -164,7 +164,7 @@ class TestCompositionYaml(unittest.TestCase):
 
     def test_composition_yaml_with_dids(self):
         """Checking that the dict is generated correctly, with DIDs."""
-        self.zc_dids.project_dids = {"DID1": {"dummy_data": {}}}
+        self.zc_dids.project_dids = {"DID1": {"type": "UInt8"}}
         self.zc_spec["Diagnostics"] = composition_yaml_with_dids.diagnostics
         with patch.object(
             CompositionYaml,