openstack/ironic-python-agent
Code Issues Proposed changes

Adding support to view indiv. cpu-core info

Browse Source 
Closes-Bug: #1639340

This commit adds the relevant changes to the get_cpu function, keeping it backwards compatible with the old method.

Change-Id: I3c3a792e88e9a041236eca7283ebfdf1026910d8
This commit is contained in:
Sharpz7 2024-07-15 12:37:37 +00:00
parent a1773199b7
commit b2ec08a15e
3 changed files with 323 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
ironic_python_agent/hardware.py
View File

@ -28,6 +28,7 @@ import shutil
import stat
import string
import time
from typing import List
from ironic_lib import utils as il_utils
from oslo_concurrency import processutils
@ -830,12 +831,26 @@ class NetworkInterface(encoding.SerializableComparable):
self.client_id = client_id
class CPUCore(encoding.SerializableComparable):
serializable_fields = ('model_name', 'frequency', 'count', 'architecture',
'flags', 'core_id')
def __init__(self, model_name, frequency, architecture,
core_id, flags=None):
self.model_name = model_name
self.frequency = frequency
self.architecture = architecture
self.core_id = core_id
self.flags = flags or []
class CPU(encoding.SerializableComparable):
serializable_fields = ('model_name', 'frequency', 'count', 'architecture',
'flags', 'socket_count')
def __init__(self, model_name, frequency, count, architecture,
flags=None, socket_count=None):
flags=None, socket_count=None, cpus: List[CPUCore] = None):
self.model_name = model_name
self.frequency = frequency
self.count = count
@ -843,6 +858,8 @@ class CPU(encoding.SerializableComparable):
self.architecture = architecture
self.flags = flags or []
self.cpus = cpus or []
class Memory(encoding.SerializableComparable):
serializable_fields = ('total', 'physical_mb')
@ -1512,35 +1529,108 @@ class GenericHardwareManager(HardwareManager):
return network_interfaces_list
def get_cpus(self):
lines = il_utils.execute('lscpu')[0]
@staticmethod
def create_cpu_info_dict(lines):
cpu_info = {k.strip().lower(): v.strip() for k, v in
(line.split(':', 1)
for line in lines.split('\n')
if line.strip())}
# Current CPU frequency can be different from maximum one on modern
# processors
freq = cpu_info.get('cpu max mhz', cpu_info.get('cpu mhz'))
for line in lines.split('\n')
if line.strip())}
flags = []
out = il_utils.try_execute('grep', '-Em1', '^flags', '/proc/cpuinfo')
if out:
try:
# Example output (much longer for a real system):
# flags : fpu vme de pse
flags = out[0].strip().split(':', 1)[1].strip().split()
except (IndexError, ValueError):
LOG.warning('Malformed CPU flags information: %s', out)
else:
LOG.warning('Failed to get CPU flags')
return cpu_info
return CPU(model_name=cpu_info.get('model name'),
frequency=freq,
# this includes hyperthreading cores
count=int(cpu_info.get('cpu(s)')),
architecture=cpu_info.get('architecture'),
flags=flags,
socket_count=int(cpu_info.get('socket(s)', 0)))
def read_cpu_info(self):
sections = []
try:
with open('/proc/cpuinfo', 'r') as file:
file_contents = file.read()
# Replace tabs with nothing (essentially removing them)
file_contents = file_contents.replace("\t", "")
# Split the string into a list of CPU core entries
# Each core's info is separated by a double newline
sections = file_contents.split("\n\n")[:-1]
except (FileNotFoundError, errors.InspectionError, OSError) as e:
LOG.warning(
'Failed to get CPU information from /proc/cpuinfo: %s', e
)
return sections
def get_cpu_cores(self):
cpu_info_dicts = []
sections = self.read_cpu_info()
for lines in sections:
cpu_info = self.create_cpu_info_dict(lines)
if cpu_info is not None:
cpu_info_dicts.append(cpu_info)
if len(cpu_info_dicts) == 0:
LOG.warning(
'No per-core CPU information found'
)
cpus = []
for cpu_info in cpu_info_dicts:
cpu = CPUCore(
model_name=cpu_info.get('model name', ''),
frequency=cpu_info.get('cpu mhz', ''),
architecture=cpu_info.get('architecture', ''),
core_id=cpu_info.get('core id', ''),
flags=cpu_info.get('flags', '').split()
)
cpus.append(cpu)
return cpus
def get_cpus(self):
lines = il_utils.execute('lscpu')[0]
cpu_info = self.create_cpu_info_dict(lines)
# NOTE(adamcarthur) Kept this assuming it was added as a fallback
# for systems where lscpu does not show flags.
if not cpu_info.get("flags", None):
sections = self.read_cpu_info()
if len(sections) == 0:
cpu_info['flags'] = ""
else:
cpu_info_proc = self.create_cpu_info_dict(sections[0])
flags = cpu_info_proc.get('flags', "")
# NOTE(adamcarthur) This is only a basic check to
# check the flags look correct
if flags and re.search(r'[A-Z!@#$%^&*()_+{}|:"<>?]', flags):
LOG.warning('Malformed CPU flags information: %s', flags)
cpu_info['flags'] = ""
else:
cpu_info['flags'] = flags
if cpu_info["flags"] == "":
LOG.warning(
'No CPU flags found'
)
return CPU(
model_name=cpu_info.get('model name', ''),
# NOTE(adamcarthur) Current CPU frequency can
# be different from maximum one on modern processors
frequency=cpu_info.get(
'cpu max mhz',
cpu_info.get('cpu mhz', "")
),
count=int(cpu_info.get('cpu(s)', 0)),
architecture=cpu_info.get('architecture', ''),
flags=cpu_info.get('flags', '').split(),
socket_count=int(cpu_info.get('socket(s)', 0)),
cpus=self.get_cpu_cores()
)
def get_memory(self):
# psutil returns a long, so we force it to an int

106
ironic_python_agent/tests/unit/samples/hardware_samples.py
View File

@ -310,6 +310,43 @@ SHRED_OUTPUT_2_ITERATIONS_ZERO_FALSE = (
)
LSCPU_OUTPUT = """
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
Address sizes: 48 bits physical, 48 bits virtual
CPU(s): 8
On-line CPU(s) list: 0-7
Thread(s) per core: 1
Core(s) per socket: 8
Socket(s): 1
NUMA node(s): 1
Vendor ID: AuthenticAMD
CPU family: 23
Model: 49
Model name: AMD EPYC 7282 16-Core Processor
Stepping: 0
CPU MHz: 2794.748
BogoMIPS: 5589.49
Hypervisor vendor: KVM
Virtualization type: full
L1d cache: 512 KiB
L1i cache: 512 KiB
L2 cache: 4 MiB
L3 cache: 16 MiB
NUMA node0 CPU(s): 0-7
Vulnerability Gather data sampling: Not affected
Vulnerability Itlb multihit: Not affected
Vulnerability L1tf: Not affected
Vulnerability Mds: Not affected
Vulnerability Meltdown: Not affected
Vulnerability Mmio stale data: Not affected
Vulnerability Retbleed: Vulnerable
Vulnerability Srbds: Not affected
Vulnerability Tsx async abort: Not affected
Flags: fpu vme de pse tsc
"""
LSCPU_OUTPUT_WITH_MAX_MHZ = """
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
@ -336,7 +373,7 @@ L3 cache: 10240K
NUMA node0 CPU(s): 0-3
"""
LSCPU_OUTPUT_NO_MAX_MHZ = """
LSCPU_OUTPUT_NO_FLAGS = """
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
@ -361,11 +398,74 @@ L3 cache: 15360K
NUMA node0 CPU(s): 0-11
"""
# NOTE(dtanstur): flags list stripped down for sanity reasons
CPUINFO_FLAGS_OUTPUT = """
PROC_CPUINFO_OUTPUT = """
processor : 0
vendor_id : AuthenticAMD
cpu family : 23
model : 49
model name : AMD EPYC 7282 16-Core Processor
stepping : 0
microcode : 0x8301055
cpu MHz : 2794.748
cache size : 512 KB
physical id : 0
siblings : 6
core id : 0
cpu cores : 6
apicid : 0
initial apicid : 0
fpu : yes
fpu_exception : yes
cpuid level : 16
wp : yes
flags : fpu vme de pse
bugs : sysret_ss_attrs
bogomips : 5589.49
TLB size : 1024 4K pages
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management:
processor : 1
vendor_id : AuthenticAMD
cpu family : 23
model : 49
model name : AMD EPYC 7282 16-Core Processor
stepping : 0
microcode : 0x8301055
cpu MHz : 2794.748
cache size : 512 KB
physical id : 0
siblings : 6
core id : 1
cpu cores : 6
apicid : 1
initial apicid : 1
fpu : yes
fpu_exception : yes
cpuid level : 16
wp : yes
flags : fpu vme de pse
bogomips : 5589.49
TLB size : 1024 4K pages
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management:
"""
ILLEGAL_PROC_FLAGS = PROC_CPUINFO_OUTPUT.replace(
"fpu vme de pse", "I am not a flag"
)
# NO PROC FLAGS should remove the lines with the word flags
NO_PROC_FLAGS = PROC_CPUINFO_OUTPUT.replace(
"flags : fpu vme de pse\n", ""
)
LSHW_JSON_OUTPUT_V1 = ("""
{
"id": "fuzzypickles",

135
ironic_python_agent/tests/unit/test_hardware.py
View File

@ -14,6 +14,7 @@
import binascii
import json
import logging
import os
import re
import shutil
@ -927,12 +928,45 @@ class TestGenericHardwareManager(base.IronicAgentTest):
'/sys/class/block/sdfake/device/vendor', 'r')
self.assertEqual('fake-vendor', vendor)
@mock.patch("builtins.open", new_callable=mock.mock_open)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus(self, mocked_execute):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''),
(hws.CPUINFO_FLAGS_OUTPUT, '')]
def test_get_cpus_max_mhz_flag_fallback(self, mocked_execute, mocked_open):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, '')]
mocked_open.side_effect = [
mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
]
with self.assertLogs(level='WARNING') as cm:
cpus = self.hardware.get_cpus()
logging.getLogger("root").warning("Test Placeholder")
self.assertEqual('AMD EPYC 7282 16-Core Processor',
cpus.model_name)
self.assertEqual('2794.748', cpus.frequency)
self.assertEqual(8, cpus.count)
self.assertEqual(1, cpus.socket_count)
self.assertEqual('x86_64', cpus.architecture)
self.assertEqual(['fpu', 'vme', 'de', 'pse', 'tsc'], cpus.flags)
self.assertEqual(["WARNING:root:Test Placeholder"], cm.output)
@mock.patch("builtins.open", new_callable=mock.mock_open)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus_max_mhz_and_flag_fallback(
self, mocked_execute, mocked_open
):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_WITH_MAX_MHZ, '')]
mocked_open.side_effect = [
mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
]
with self.assertLogs(level='WARNING') as cm:
cpus = self.hardware.get_cpus()
logging.getLogger("root").warning("Test Placeholder")
cpus = self.hardware.get_cpus()
self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz',
cpus.model_name)
self.assertEqual('2400.0000', cpus.frequency)
@ -941,46 +975,85 @@ class TestGenericHardwareManager(base.IronicAgentTest):
self.assertEqual('x86_64', cpus.architecture)
self.assertEqual(['fpu', 'vme', 'de', 'pse'], cpus.flags)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus2(self, mocked_execute):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_MAX_MHZ, ''),
(hws.CPUINFO_FLAGS_OUTPUT, '')]
self.assertEqual(["WARNING:root:Test Placeholder"], cm.output)
@mock.patch("builtins.open", new_callable=mock.mock_open)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus_multi(self, mocked_execute, mocked_open):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, '')]
mocked_open.side_effect = [
mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
]
with self.assertLogs(level='WARNING') as cm:
cpus = self.hardware.get_cpus()
logging.getLogger("root").warning("Test Placeholder")
clock_speeds = ["2794.748", "2794.748"]
core_ids = [0, 1, 2, 3, 4, 5, 6, 7]
self.assertGreater(len(cpus.cpus), 0)
for i, cpu in enumerate(cpus.cpus):
self.assertEqual('AMD EPYC 7282 16-Core Processor',
cpu.model_name)
self.assertEqual(clock_speeds[i], cpu.frequency)
self.assertEqual(str(core_ids[i]), cpu.core_id)
self.assertEqual(8, cpus.count)
self.assertEqual(1, cpus.socket_count)
self.assertEqual('x86_64', cpus.architecture)
self.assertEqual(['fpu', 'vme', 'de', 'pse', 'tsc'], cpus.flags)
self.assertEqual(["WARNING:root:Test Placeholder"], cm.output)
@mock.patch("builtins.open", new_callable=mock.mock_open)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus_no_flags(self, mocked_execute, mocked_open):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_FLAGS, '')]
mocked_open.side_effect = [
mock.mock_open(read_data=hws.NO_PROC_FLAGS).return_value,
mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
]
with self.assertLogs(level='WARNING') as cm:
cpus = self.hardware.get_cpus()
cpus = self.hardware.get_cpus()
self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz',
cpus.model_name)
self.assertEqual('1794.433', cpus.frequency)
self.assertEqual(12, cpus.count)
self.assertEqual(1, cpus.socket_count)
self.assertEqual('x86_64', cpus.architecture)
self.assertEqual(['fpu', 'vme', 'de', 'pse'], cpus.flags)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus_no_flags(self, mocked_execute):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''),
processutils.ProcessExecutionError()]
cpus = self.hardware.get_cpus()
self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz',
cpus.model_name)
self.assertEqual('2400.0000', cpus.frequency)
self.assertEqual(4, cpus.count)
self.assertEqual('x86_64', cpus.architecture)
self.assertEqual([], cpus.flags)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus_illegal_flags(self, mocked_execute):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT, ''),
('I am not a flag', '')]
self.assertEqual(["WARNING:root:No CPU flags found"], cm.output)
cpus = self.hardware.get_cpus()
self.assertEqual('Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz',
@mock.patch("builtins.open", new_callable=mock.mock_open)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_cpus_illegal_flags(self, mocked_execute, mocked_open):
mocked_execute.side_effect = [(hws.LSCPU_OUTPUT_NO_FLAGS, '')]
mocked_open.side_effect = [
mock.mock_open(read_data=hws.ILLEGAL_PROC_FLAGS).return_value,
mock.mock_open(read_data=hws.PROC_CPUINFO_OUTPUT).return_value,
]
with self.assertLogs(level='WARNING') as cm:
cpus = self.hardware.get_cpus()
self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz',
cpus.model_name)
self.assertEqual('2400.0000', cpus.frequency)
self.assertEqual(4, cpus.count)
self.assertEqual('1794.433', cpus.frequency)
self.assertEqual(12, cpus.count)
self.assertEqual('x86_64', cpus.architecture)
self.assertEqual([], cpus.flags)
# Check if the warning was logged
self.assertEqual([
"WARNING:root:Malformed CPU flags information: I am not a flag",
"WARNING:root:No CPU flags found"], cm.output)
@mock.patch('psutil.virtual_memory', autospec=True)
@mock.patch.object(il_utils, 'execute', autospec=True)
def test_get_memory_psutil_v1(self, mocked_execute, mocked_psutil):