Adding support to view indiv. cpu-core info

Closes-Bug: #1639340

This commit adds the relevant changes to the get_cpu function, keeping it backwards compatible with the old method.

Change-Id: I3c3a792e88e9a041236eca7283ebfdf1026910d8

ironic_python_agent/hardware.py

@@ -28,6 +28,7 @@ import shutil
 import stat
 import string
 import time
+from typing import List
 
 from ironic_lib import utils as il_utils
 from oslo_concurrency import processutils
@@ -830,12 +831,26 @@ class NetworkInterface(encoding.SerializableComparable):
         self.client_id = client_id
 
 
+class CPUCore(encoding.SerializableComparable):
+    serializable_fields = ('model_name', 'frequency', 'count', 'architecture',
+                           'flags', 'core_id')
+
+    def __init__(self, model_name, frequency, architecture,
+                 core_id, flags=None):
+        self.model_name = model_name
+        self.frequency = frequency
+        self.architecture = architecture
+        self.core_id = core_id
+
+        self.flags = flags or []
+
+
 class CPU(encoding.SerializableComparable):
     serializable_fields = ('model_name', 'frequency', 'count', 'architecture',
                            'flags', 'socket_count')
 
     def __init__(self, model_name, frequency, count, architecture,
-                 flags=None, socket_count=None):
+                 flags=None, socket_count=None, cpus: List[CPUCore] = None):
         self.model_name = model_name
         self.frequency = frequency
         self.count = count
@@ -843,6 +858,8 @@ class CPU(encoding.SerializableComparable):
         self.architecture = architecture
         self.flags = flags or []
 
+        self.cpus = cpus or []
+
 
 class Memory(encoding.SerializableComparable):
     serializable_fields = ('total', 'physical_mb')
@@ -1512,35 +1529,108 @@ class GenericHardwareManager(HardwareManager):
 
         return network_interfaces_list
 
-    def get_cpus(self):
+    @staticmethod
-        lines = il_utils.execute('lscpu')[0]
+    def create_cpu_info_dict(lines):
         cpu_info = {k.strip().lower(): v.strip() for k, v in
                     (line.split(':', 1)
-                     for line in lines.split('\n')
+                    for line in lines.split('\n')
-                     if line.strip())}
+                    if line.strip())}
-        # Current CPU frequency can be different from maximum one on modern
-        # processors
-        freq = cpu_info.get('cpu max mhz', cpu_info.get('cpu mhz'))
 
-        flags = []
+        return cpu_info
-        out = il_utils.try_execute('grep', '-Em1', '^flags', '/proc/cpuinfo')
-        if out:
-            try:
-                # Example output (much longer for a real system):
-                # flags           : fpu vme de pse
-                flags = out[0].strip().split(':', 1)[1].strip().split()
-            except (IndexError, ValueError):
-                LOG.warning('Malformed CPU flags information: %s', out)
-        else:
-            LOG.warning('Failed to get CPU flags')
 
-        return CPU(model_name=cpu_info.get('model name'),
+    def read_cpu_info(self):
-                   frequency=freq,
+        sections = []
-                   # this includes hyperthreading cores
+
-                   count=int(cpu_info.get('cpu(s)')),
+        try:
-                   architecture=cpu_info.get('architecture'),
+            with open('/proc/cpuinfo', 'r') as file:
-                   flags=flags,
+                file_contents = file.read()
-                   socket_count=int(cpu_info.get('socket(s)', 0)))
+
+            # Replace tabs with nothing (essentially removing them)
+            file_contents = file_contents.replace("\t", "")
+
+            # Split the string into a list of CPU core entries
+            # Each core's info is separated by a double newline
+            sections = file_contents.split("\n\n")[:-1]
+
+        except (FileNotFoundError, errors.InspectionError, OSError) as e:
+            LOG.warning(
+                'Failed to get CPU information from /proc/cpuinfo: %s', e
+            )
+
+        return sections
+
+    def get_cpu_cores(self):
+        cpu_info_dicts = []
+
+        sections = self.read_cpu_info()
+
+        for lines in sections:
+            cpu_info = self.create_cpu_info_dict(lines)
+
+            if cpu_info is not None:
+                cpu_info_dicts.append(cpu_info)
+
+        if len(cpu_info_dicts) == 0:
+            LOG.warning(
+                'No per-core CPU information found'
+            )
+
+        cpus = []
+        for cpu_info in cpu_info_dicts:
+            cpu = CPUCore(
+                model_name=cpu_info.get('model name', ''),
+                frequency=cpu_info.get('cpu mhz', ''),
+                architecture=cpu_info.get('architecture', ''),
+                core_id=cpu_info.get('core id', ''),
+                flags=cpu_info.get('flags', '').split()
+            )
+            cpus.append(cpu)
+
+        return cpus
+
+    def get_cpus(self):
+        lines = il_utils.execute('lscpu')[0]
+        cpu_info = self.create_cpu_info_dict(lines)
+
+        # NOTE(adamcarthur) Kept this assuming it was added as a fallback
+        # for systems where lscpu does not show flags.
+        if not cpu_info.get("flags", None):
+
+            sections = self.read_cpu_info()
+            if len(sections) == 0:
+                cpu_info['flags'] = ""
+            else:
+                cpu_info_proc = self.create_cpu_info_dict(sections[0])
+
+                flags = cpu_info_proc.get('flags', "")
+
+                # NOTE(adamcarthur) This is only a basic check to
+                # check the flags look correct
+                if flags and re.search(r'[A-Z!@#$%^&*()_+{}|:"<>?]', flags):
+                    LOG.warning('Malformed CPU flags information: %s', flags)
+                    cpu_info['flags'] = ""
+                else:
+                    cpu_info['flags'] = flags
+
+        if cpu_info["flags"] == "":
+            LOG.warning(
+                'No CPU flags found'
+            )
+
+        return CPU(
+            model_name=cpu_info.get('model name', ''),
+            # NOTE(adamcarthur) Current CPU frequency can
+            # be different from maximum one on modern processors
+            frequency=cpu_info.get(
+                'cpu max mhz',
+                cpu_info.get('cpu mhz', "")
+            ),
+            count=int(cpu_info.get('cpu(s)', 0)),
+            architecture=cpu_info.get('architecture', ''),
+            flags=cpu_info.get('flags', '').split(),
+            socket_count=int(cpu_info.get('socket(s)', 0)),
+            cpus=self.get_cpu_cores()
+        )
 
     def get_memory(self):
         # psutil returns a long, so we force it to an int

ironic_python_agent/tests/unit/samples/hardware_samples.py

@@ -310,6 +310,43 @@ SHRED_OUTPUT_2_ITERATIONS_ZERO_FALSE = (
 )
 
 LSCPU_OUTPUT = """
+Architecture:                       x86_64
+CPU op-mode(s):                     32-bit, 64-bit
+Byte Order:                         Little Endian
+Address sizes:                      48 bits physical, 48 bits virtual
+CPU(s):                             8
+On-line CPU(s) list:                0-7
+Thread(s) per core:                 1
+Core(s) per socket:                 8
+Socket(s):                          1
+NUMA node(s):                       1
+Vendor ID:                          AuthenticAMD
+CPU family:                         23
+Model:                              49
+Model name:                         AMD EPYC 7282 16-Core Processor
+Stepping:                           0
+CPU MHz:                            2794.748
+BogoMIPS:                           5589.49
+Hypervisor vendor:                  KVM
+Virtualization type:                full
+L1d cache:                          512 KiB
+L1i cache:                          512 KiB
+L2 cache:                           4 MiB
+L3 cache:                           16 MiB
+NUMA node0 CPU(s):                  0-7
+Vulnerability Gather data sampling: Not affected
+Vulnerability Itlb multihit:        Not affected
+Vulnerability L1tf:                 Not affected
+Vulnerability Mds:                  Not affected
+Vulnerability Meltdown:             Not affected
+Vulnerability Mmio stale data:      Not affected
+Vulnerability Retbleed:             Vulnerable
+Vulnerability Srbds:                Not affected
+Vulnerability Tsx async abort:      Not affected
+Flags:                              fpu vme de pse tsc
+"""
+
+LSCPU_OUTPUT_WITH_MAX_MHZ = """
 Architecture:          x86_64
 CPU op-mode(s):        32-bit, 64-bit
 Byte Order:            Little Endian
@@ -336,7 +373,7 @@ L3 cache:              10240K
 NUMA node0 CPU(s):     0-3
 """
 
-LSCPU_OUTPUT_NO_MAX_MHZ = """
+LSCPU_OUTPUT_NO_FLAGS = """
 Architecture:          x86_64
 CPU op-mode(s):        32-bit, 64-bit
 Byte Order:            Little Endian
@@ -361,11 +398,74 @@ L3 cache:              15360K
 NUMA node0 CPU(s):     0-11
 """
 
-# NOTE(dtanstur): flags list stripped down for sanity reasons
+
-CPUINFO_FLAGS_OUTPUT = """
+PROC_CPUINFO_OUTPUT = """
+processor       : 0
+vendor_id       : AuthenticAMD
+cpu family      : 23
+model           : 49
+model name      : AMD EPYC 7282 16-Core Processor
+stepping        : 0
+microcode       : 0x8301055
+cpu MHz         : 2794.748
+cache size      : 512 KB
+physical id     : 0
+siblings        : 6
+core id         : 0
+cpu cores       : 6
+apicid          : 0
+initial apicid  : 0
+fpu             : yes
+fpu_exception   : yes
+cpuid level     : 16
+wp              : yes
 flags           : fpu vme de pse
+bugs            : sysret_ss_attrs
+bogomips        : 5589.49
+TLB size        : 1024 4K pages
+clflush size    : 64
+cache_alignment : 64
+address sizes   : 40 bits physical, 48 bits virtual
+power management:
+
+processor       : 1
+vendor_id       : AuthenticAMD
+cpu family      : 23
+model           : 49
+model name      : AMD EPYC 7282 16-Core Processor
+stepping        : 0
+microcode       : 0x8301055
+cpu MHz         : 2794.748
+cache size      : 512 KB
+physical id     : 0
+siblings        : 6
+core id         : 1
+cpu cores       : 6
+apicid          : 1
+initial apicid  : 1
+fpu             : yes
+fpu_exception   : yes
+cpuid level     : 16
+wp              : yes
+flags           : fpu vme de pse
+bogomips        : 5589.49
+TLB size        : 1024 4K pages
+clflush size    : 64
+cache_alignment : 64
+address sizes   : 40 bits physical, 48 bits virtual
+power management:
+
 """
 
+ILLEGAL_PROC_FLAGS = PROC_CPUINFO_OUTPUT.replace(
+    "fpu vme de pse", "I am not a flag"
+)
+
+# NO PROC FLAGS should remove the lines with the word flags
+NO_PROC_FLAGS = PROC_CPUINFO_OUTPUT.replace(
+    "flags           : fpu vme de pse\n", ""
+)
+
 LSHW_JSON_OUTPUT_V1 = ("""
 {
   "id": "fuzzypickles",

ironic_python_agent/tests/unit/test_hardware.py

@@ -14,6 +14,7 @@
 
 import binascii
 import json
+import logging
 import os
 import re
 import shutil
@@ -927,12 +928,45 @@ class TestGenericHardwareManager(base.IronicAgentTest):
                 '/sys/class/block/sdfake/device/vendor', 'r')
             self.assertEqual('fake-vendor', vendor)
 
+    @mock.patch("builtins.open", new_callable=mock.mock_open)
     @mock.patch.object(il_utils, 'execute', autospec=True)
-    def test_get_cpus(self, mocked_execute):
+    def test_get_cpus_max_mhz_flag_fallback(self, mocked_execute, mocked_open):
-        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''),
+        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, '')]
-                                      (hws.CPUINFO_FLAGS_OUTPUT, '')]
+
+        mocked_open.side_effect = [
+            mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
+        ]
+
+        with self.assertLogs(level='WARNING') as cm:
+            cpus = self.hardware.get_cpus()
+            logging.getLogger("root").warning("Test Placeholder")
+
+        self.assertEqual('AMD EPYC 7282 16-Core Processor',
+                         cpus.model_name)
+        self.assertEqual('2794.748', cpus.frequency)
+        self.assertEqual(8, cpus.count)
+        self.assertEqual(1, cpus.socket_count)
+        self.assertEqual('x86_64', cpus.architecture)
+        self.assertEqual(['fpu', 'vme', 'de', 'pse', 'tsc'], cpus.flags)
+
+        self.assertEqual(["WARNING:root:Test Placeholder"], cm.output)
+
+    @mock.patch("builtins.open", new_callable=mock.mock_open)
+    @mock.patch.object(il_utils, 'execute', autospec=True)
+    def test_get_cpus_max_mhz_and_flag_fallback(
+        self, mocked_execute, mocked_open
+    ):
+        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_WITH_MAX_MHZ, '')]
+
+        mocked_open.side_effect = [
+            mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
+            mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
+        ]
+
+        with self.assertLogs(level='WARNING') as cm:
+            cpus = self.hardware.get_cpus()
+            logging.getLogger("root").warning("Test Placeholder")
 
-        cpus = self.hardware.get_cpus()
         self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz',
                          cpus.model_name)
         self.assertEqual('2400.0000', cpus.frequency)
@@ -941,46 +975,85 @@ class TestGenericHardwareManager(base.IronicAgentTest):
         self.assertEqual('x86_64', cpus.architecture)
         self.assertEqual(['fpu', 'vme', 'de', 'pse'], cpus.flags)
 
-    @mock.patch.object(il_utils, 'execute', autospec=True)
+        self.assertEqual(["WARNING:root:Test Placeholder"], cm.output)
-    def test_get_cpus2(self, mocked_execute):
+
-        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_MAX_MHZ, ''),
+    @mock.patch("builtins.open", new_callable=mock.mock_open)
-                                      (hws.CPUINFO_FLAGS_OUTPUT, '')]
+    @mock.patch.object(il_utils, 'execute', autospec=True)
+    def test_get_cpus_multi(self, mocked_execute, mocked_open):
+        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, '')]
+        mocked_open.side_effect = [
+            mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
+        ]
+
+        with self.assertLogs(level='WARNING') as cm:
+            cpus = self.hardware.get_cpus()
+            logging.getLogger("root").warning("Test Placeholder")
+
+        clock_speeds = ["2794.748", "2794.748"]
+        core_ids = [0, 1, 2, 3, 4, 5, 6, 7]
+
+        self.assertGreater(len(cpus.cpus), 0)
+
+        for i, cpu in enumerate(cpus.cpus):
+            self.assertEqual('AMD EPYC 7282 16-Core Processor',
+                             cpu.model_name)
+
+            self.assertEqual(clock_speeds[i], cpu.frequency)
+            self.assertEqual(str(core_ids[i]), cpu.core_id)
+
+        self.assertEqual(8, cpus.count)
+        self.assertEqual(1, cpus.socket_count)
+        self.assertEqual('x86_64', cpus.architecture)
+        self.assertEqual(['fpu', 'vme', 'de', 'pse', 'tsc'], cpus.flags)
+
+        self.assertEqual(["WARNING:root:Test Placeholder"], cm.output)
+
+    @mock.patch("builtins.open", new_callable=mock.mock_open)
+    @mock.patch.object(il_utils, 'execute', autospec=True)
+    def test_get_cpus_no_flags(self, mocked_execute, mocked_open):
+        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_FLAGS, '')]
+
+        mocked_open.side_effect = [
+            mock.mock_open(read_data=hws.NO_PROC_FLAGS).return_value,
+            mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
+        ]
+
+        with self.assertLogs(level='WARNING') as cm:
+            cpus = self.hardware.get_cpus()
 
-        cpus = self.hardware.get_cpus()
         self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz',
                          cpus.model_name)
         self.assertEqual('1794.433', cpus.frequency)
         self.assertEqual(12, cpus.count)
-        self.assertEqual(1, cpus.socket_count)
-        self.assertEqual('x86_64', cpus.architecture)
-        self.assertEqual(['fpu', 'vme', 'de', 'pse'], cpus.flags)
-
-    @mock.patch.object(il_utils, 'execute', autospec=True)
-    def test_get_cpus_no_flags(self, mocked_execute):
-        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''),
-                                      processutils.ProcessExecutionError()]
-
-        cpus = self.hardware.get_cpus()
-        self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz',
-                         cpus.model_name)
-        self.assertEqual('2400.0000', cpus.frequency)
-        self.assertEqual(4, cpus.count)
         self.assertEqual('x86_64', cpus.architecture)
         self.assertEqual([], cpus.flags)
 
-    @mock.patch.object(il_utils, 'execute', autospec=True)
+        self.assertEqual(["WARNING:root:No CPU flags found"], cm.output)
-    def test_get_cpus_illegal_flags(self, mocked_execute):
-        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''),
-                                      ('I am not a flag', '')]
 
-        cpus = self.hardware.get_cpus()
+    @mock.patch("builtins.open", new_callable=mock.mock_open)
-        self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz',
+    @mock.patch.object(il_utils, 'execute', autospec=True)
+    def test_get_cpus_illegal_flags(self, mocked_execute, mocked_open):
+        mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_FLAGS, '')]
+        mocked_open.side_effect = [
+            mock.mock_open(read_data=hws.ILLEGAL_PROC_FLAGS).return_value,
+            mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
+        ]
+
+        with self.assertLogs(level='WARNING') as cm:
+            cpus = self.hardware.get_cpus()
+
+        self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz',
                          cpus.model_name)
-        self.assertEqual('2400.0000', cpus.frequency)
+        self.assertEqual('1794.433', cpus.frequency)
-        self.assertEqual(4, cpus.count)
+        self.assertEqual(12, cpus.count)
         self.assertEqual('x86_64', cpus.architecture)
         self.assertEqual([], cpus.flags)
 
+        # Check if the warning was logged
+        self.assertEqual([
+            "WARNING:root:Malformed CPU flags information: I am not a flag",
+            "WARNING:root:No CPU flags found"], cm.output)
+
     @mock.patch('psutil.virtual_memory', autospec=True)
     @mock.patch.object(il_utils, 'execute', autospec=True)
     def test_get_memory_psutil_v1(self, mocked_execute, mocked_psutil):