diff --git a/ironic_python_agent/hardware.py b/ironic_python_agent/hardware.py index 9fd237323..3ea1e45c7 100644 --- a/ironic_python_agent/hardware.py +++ b/ironic_python_agent/hardware.py @@ -28,6 +28,7 @@ import shutil import stat import string import time +from typing import List from ironic_lib import utils as il_utils from oslo_concurrency import processutils @@ -830,12 +831,26 @@ class NetworkInterface(encoding.SerializableComparable): self.client_id = client_id +class CPUCore(encoding.SerializableComparable): + serializable_fields = ('model_name', 'frequency', 'count', 'architecture', + 'flags', 'core_id') + + def __init__(self, model_name, frequency, architecture, + core_id, flags=None): + self.model_name = model_name + self.frequency = frequency + self.architecture = architecture + self.core_id = core_id + + self.flags = flags or [] + + class CPU(encoding.SerializableComparable): serializable_fields = ('model_name', 'frequency', 'count', 'architecture', 'flags', 'socket_count') def __init__(self, model_name, frequency, count, architecture, - flags=None, socket_count=None): + flags=None, socket_count=None, cpus: List[CPUCore] = None): self.model_name = model_name self.frequency = frequency self.count = count @@ -843,6 +858,8 @@ class CPU(encoding.SerializableComparable): self.architecture = architecture self.flags = flags or [] + self.cpus = cpus or [] + class Memory(encoding.SerializableComparable): serializable_fields = ('total', 'physical_mb') @@ -1512,35 +1529,108 @@ class GenericHardwareManager(HardwareManager): return network_interfaces_list - def get_cpus(self): - lines = il_utils.execute('lscpu')[0] + @staticmethod + def create_cpu_info_dict(lines): cpu_info = {k.strip().lower(): v.strip() for k, v in (line.split(':', 1) - for line in lines.split('\n') - if line.strip())} - # Current CPU frequency can be different from maximum one on modern - # processors - freq = cpu_info.get('cpu max mhz', cpu_info.get('cpu mhz')) + for line in lines.split('\n') + if line.strip())} - flags = [] - out = il_utils.try_execute('grep', '-Em1', '^flags', '/proc/cpuinfo') - if out: - try: - # Example output (much longer for a real system): - # flags : fpu vme de pse - flags = out[0].strip().split(':', 1)[1].strip().split() - except (IndexError, ValueError): - LOG.warning('Malformed CPU flags information: %s', out) - else: - LOG.warning('Failed to get CPU flags') + return cpu_info - return CPU(model_name=cpu_info.get('model name'), - frequency=freq, - # this includes hyperthreading cores - count=int(cpu_info.get('cpu(s)')), - architecture=cpu_info.get('architecture'), - flags=flags, - socket_count=int(cpu_info.get('socket(s)', 0))) + def read_cpu_info(self): + sections = [] + + try: + with open('/proc/cpuinfo', 'r') as file: + file_contents = file.read() + + # Replace tabs with nothing (essentially removing them) + file_contents = file_contents.replace("\t", "") + + # Split the string into a list of CPU core entries + # Each core's info is separated by a double newline + sections = file_contents.split("\n\n")[:-1] + + except (FileNotFoundError, errors.InspectionError, OSError) as e: + LOG.warning( + 'Failed to get CPU information from /proc/cpuinfo: %s', e + ) + + return sections + + def get_cpu_cores(self): + cpu_info_dicts = [] + + sections = self.read_cpu_info() + + for lines in sections: + cpu_info = self.create_cpu_info_dict(lines) + + if cpu_info is not None: + cpu_info_dicts.append(cpu_info) + + if len(cpu_info_dicts) == 0: + LOG.warning( + 'No per-core CPU information found' + ) + + cpus = [] + for cpu_info in cpu_info_dicts: + cpu = CPUCore( + model_name=cpu_info.get('model name', ''), + frequency=cpu_info.get('cpu mhz', ''), + architecture=cpu_info.get('architecture', ''), + core_id=cpu_info.get('core id', ''), + flags=cpu_info.get('flags', '').split() + ) + cpus.append(cpu) + + return cpus + + def get_cpus(self): + lines = il_utils.execute('lscpu')[0] + cpu_info = self.create_cpu_info_dict(lines) + + # NOTE(adamcarthur) Kept this assuming it was added as a fallback + # for systems where lscpu does not show flags. + if not cpu_info.get("flags", None): + + sections = self.read_cpu_info() + if len(sections) == 0: + cpu_info['flags'] = "" + else: + cpu_info_proc = self.create_cpu_info_dict(sections[0]) + + flags = cpu_info_proc.get('flags', "") + + # NOTE(adamcarthur) This is only a basic check to + # check the flags look correct + if flags and re.search(r'[A-Z!@#$%^&*()_+{}|:"<>?]', flags): + LOG.warning('Malformed CPU flags information: %s', flags) + cpu_info['flags'] = "" + else: + cpu_info['flags'] = flags + + if cpu_info["flags"] == "": + LOG.warning( + 'No CPU flags found' + ) + + return CPU( + model_name=cpu_info.get('model name', ''), + # NOTE(adamcarthur) Current CPU frequency can + # be different from maximum one on modern processors + frequency=cpu_info.get( + 'cpu max mhz', + cpu_info.get('cpu mhz', "") + ), + count=int(cpu_info.get('cpu(s)', 0)), + architecture=cpu_info.get('architecture', ''), + flags=cpu_info.get('flags', '').split(), + socket_count=int(cpu_info.get('socket(s)', 0)), + cpus=self.get_cpu_cores() + ) def get_memory(self): # psutil returns a long, so we force it to an int diff --git a/ironic_python_agent/tests/unit/samples/hardware_samples.py b/ironic_python_agent/tests/unit/samples/hardware_samples.py index 914485f41..730127519 100644 --- a/ironic_python_agent/tests/unit/samples/hardware_samples.py +++ b/ironic_python_agent/tests/unit/samples/hardware_samples.py @@ -310,6 +310,43 @@ SHRED_OUTPUT_2_ITERATIONS_ZERO_FALSE = ( ) LSCPU_OUTPUT = """ +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Byte Order: Little Endian +Address sizes: 48 bits physical, 48 bits virtual +CPU(s): 8 +On-line CPU(s) list: 0-7 +Thread(s) per core: 1 +Core(s) per socket: 8 +Socket(s): 1 +NUMA node(s): 1 +Vendor ID: AuthenticAMD +CPU family: 23 +Model: 49 +Model name: AMD EPYC 7282 16-Core Processor +Stepping: 0 +CPU MHz: 2794.748 +BogoMIPS: 5589.49 +Hypervisor vendor: KVM +Virtualization type: full +L1d cache: 512 KiB +L1i cache: 512 KiB +L2 cache: 4 MiB +L3 cache: 16 MiB +NUMA node0 CPU(s): 0-7 +Vulnerability Gather data sampling: Not affected +Vulnerability Itlb multihit: Not affected +Vulnerability L1tf: Not affected +Vulnerability Mds: Not affected +Vulnerability Meltdown: Not affected +Vulnerability Mmio stale data: Not affected +Vulnerability Retbleed: Vulnerable +Vulnerability Srbds: Not affected +Vulnerability Tsx async abort: Not affected +Flags: fpu vme de pse tsc +""" + +LSCPU_OUTPUT_WITH_MAX_MHZ = """ Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian @@ -336,7 +373,7 @@ L3 cache: 10240K NUMA node0 CPU(s): 0-3 """ -LSCPU_OUTPUT_NO_MAX_MHZ = """ +LSCPU_OUTPUT_NO_FLAGS = """ Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian @@ -361,11 +398,74 @@ L3 cache: 15360K NUMA node0 CPU(s): 0-11 """ -# NOTE(dtanstur): flags list stripped down for sanity reasons -CPUINFO_FLAGS_OUTPUT = """ + +PROC_CPUINFO_OUTPUT = """ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 23 +model : 49 +model name : AMD EPYC 7282 16-Core Processor +stepping : 0 +microcode : 0x8301055 +cpu MHz : 2794.748 +cache size : 512 KB +physical id : 0 +siblings : 6 +core id : 0 +cpu cores : 6 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes flags : fpu vme de pse +bugs : sysret_ss_attrs +bogomips : 5589.49 +TLB size : 1024 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 40 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : AuthenticAMD +cpu family : 23 +model : 49 +model name : AMD EPYC 7282 16-Core Processor +stepping : 0 +microcode : 0x8301055 +cpu MHz : 2794.748 +cache size : 512 KB +physical id : 0 +siblings : 6 +core id : 1 +cpu cores : 6 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse +bogomips : 5589.49 +TLB size : 1024 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 40 bits physical, 48 bits virtual +power management: + """ +ILLEGAL_PROC_FLAGS = PROC_CPUINFO_OUTPUT.replace( + "fpu vme de pse", "I am not a flag" +) + +# NO PROC FLAGS should remove the lines with the word flags +NO_PROC_FLAGS = PROC_CPUINFO_OUTPUT.replace( + "flags : fpu vme de pse\n", "" +) + LSHW_JSON_OUTPUT_V1 = (""" { "id": "fuzzypickles", diff --git a/ironic_python_agent/tests/unit/test_hardware.py b/ironic_python_agent/tests/unit/test_hardware.py index 75869971a..469d96254 100644 --- a/ironic_python_agent/tests/unit/test_hardware.py +++ b/ironic_python_agent/tests/unit/test_hardware.py @@ -14,6 +14,7 @@ import binascii import json +import logging import os import re import shutil @@ -927,12 +928,45 @@ class TestGenericHardwareManager(base.IronicAgentTest): '/sys/class/block/sdfake/device/vendor', 'r') self.assertEqual('fake-vendor', vendor) + @mock.patch("builtins.open", new_callable=mock.mock_open) @mock.patch.object(il_utils, 'execute', autospec=True) - def test_get_cpus(self, mocked_execute): - mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''), - (hws.CPUINFO_FLAGS_OUTPUT, '')] + def test_get_cpus_max_mhz_flag_fallback(self, mocked_execute, mocked_open): + mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, '')] + + mocked_open.side_effect = [ + mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value, + ] + + with self.assertLogs(level='WARNING') as cm: + cpus = self.hardware.get_cpus() + logging.getLogger("root").warning("Test Placeholder") + + self.assertEqual('AMD EPYC 7282 16-Core Processor', + cpus.model_name) + self.assertEqual('2794.748', cpus.frequency) + self.assertEqual(8, cpus.count) + self.assertEqual(1, cpus.socket_count) + self.assertEqual('x86_64', cpus.architecture) + self.assertEqual(['fpu', 'vme', 'de', 'pse', 'tsc'], cpus.flags) + + self.assertEqual(["WARNING:root:Test Placeholder"], cm.output) + + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch.object(il_utils, 'execute', autospec=True) + def test_get_cpus_max_mhz_and_flag_fallback( + self, mocked_execute, mocked_open + ): + mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_WITH_MAX_MHZ, '')] + + mocked_open.side_effect = [ + mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value, + mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value, + ] + + with self.assertLogs(level='WARNING') as cm: + cpus = self.hardware.get_cpus() + logging.getLogger("root").warning("Test Placeholder") - cpus = self.hardware.get_cpus() self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz', cpus.model_name) self.assertEqual('2400.0000', cpus.frequency) @@ -941,46 +975,85 @@ class TestGenericHardwareManager(base.IronicAgentTest): self.assertEqual('x86_64', cpus.architecture) self.assertEqual(['fpu', 'vme', 'de', 'pse'], cpus.flags) - @mock.patch.object(il_utils, 'execute', autospec=True) - def test_get_cpus2(self, mocked_execute): - mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_MAX_MHZ, ''), - (hws.CPUINFO_FLAGS_OUTPUT, '')] + self.assertEqual(["WARNING:root:Test Placeholder"], cm.output) + + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch.object(il_utils, 'execute', autospec=True) + def test_get_cpus_multi(self, mocked_execute, mocked_open): + mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, '')] + mocked_open.side_effect = [ + mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value, + ] + + with self.assertLogs(level='WARNING') as cm: + cpus = self.hardware.get_cpus() + logging.getLogger("root").warning("Test Placeholder") + + clock_speeds = ["2794.748", "2794.748"] + core_ids = [0, 1, 2, 3, 4, 5, 6, 7] + + self.assertGreater(len(cpus.cpus), 0) + + for i, cpu in enumerate(cpus.cpus): + self.assertEqual('AMD EPYC 7282 16-Core Processor', + cpu.model_name) + + self.assertEqual(clock_speeds[i], cpu.frequency) + self.assertEqual(str(core_ids[i]), cpu.core_id) + + self.assertEqual(8, cpus.count) + self.assertEqual(1, cpus.socket_count) + self.assertEqual('x86_64', cpus.architecture) + self.assertEqual(['fpu', 'vme', 'de', 'pse', 'tsc'], cpus.flags) + + self.assertEqual(["WARNING:root:Test Placeholder"], cm.output) + + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch.object(il_utils, 'execute', autospec=True) + def test_get_cpus_no_flags(self, mocked_execute, mocked_open): + mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_FLAGS, '')] + + mocked_open.side_effect = [ + mock.mock_open(read_data=hws.NO_PROC_FLAGS).return_value, + mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value, + ] + + with self.assertLogs(level='WARNING') as cm: + cpus = self.hardware.get_cpus() - cpus = self.hardware.get_cpus() self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz', cpus.model_name) self.assertEqual('1794.433', cpus.frequency) self.assertEqual(12, cpus.count) - self.assertEqual(1, cpus.socket_count) - self.assertEqual('x86_64', cpus.architecture) - self.assertEqual(['fpu', 'vme', 'de', 'pse'], cpus.flags) - - @mock.patch.object(il_utils, 'execute', autospec=True) - def test_get_cpus_no_flags(self, mocked_execute): - mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''), - processutils.ProcessExecutionError()] - - cpus = self.hardware.get_cpus() - self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz', - cpus.model_name) - self.assertEqual('2400.0000', cpus.frequency) - self.assertEqual(4, cpus.count) self.assertEqual('x86_64', cpus.architecture) self.assertEqual([], cpus.flags) - @mock.patch.object(il_utils, 'execute', autospec=True) - def test_get_cpus_illegal_flags(self, mocked_execute): - mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''), - ('I am not a flag', '')] + self.assertEqual(["WARNING:root:No CPU flags found"], cm.output) - cpus = self.hardware.get_cpus() - self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz', + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch.object(il_utils, 'execute', autospec=True) + def test_get_cpus_illegal_flags(self, mocked_execute, mocked_open): + mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_FLAGS, '')] + mocked_open.side_effect = [ + mock.mock_open(read_data=hws.ILLEGAL_PROC_FLAGS).return_value, + mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value, + ] + + with self.assertLogs(level='WARNING') as cm: + cpus = self.hardware.get_cpus() + + self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz', cpus.model_name) - self.assertEqual('2400.0000', cpus.frequency) - self.assertEqual(4, cpus.count) + self.assertEqual('1794.433', cpus.frequency) + self.assertEqual(12, cpus.count) self.assertEqual('x86_64', cpus.architecture) self.assertEqual([], cpus.flags) + # Check if the warning was logged + self.assertEqual([ + "WARNING:root:Malformed CPU flags information: I am not a flag", + "WARNING:root:No CPU flags found"], cm.output) + @mock.patch('psutil.virtual_memory', autospec=True) @mock.patch.object(il_utils, 'execute', autospec=True) def test_get_memory_psutil_v1(self, mocked_execute, mocked_psutil):