openstack/ironic-python-agent
Code Issues Proposed changes

Extend hardware manager with data needed for inspector

Browse Source 
* Added NetworkInterface.ip4_address
* Added HardwareManager.get_bmc_address()
* Added Memory.physical_mb

  This is total memory as reported by dmidecode, and yes,
  it's different from total, as it includes kernel reserved space.

* Added CPU.architecture

  As a side effect, get_cpus was switched to lscpu.
  Also fixes problem when get_cpus reported the current frequency
  instead of maximum one.

Change-Id: I4080d4d551eb0bb995a94ef9a300351910c09fb9
This commit is contained in:
Dmitry Tantsur
2015-08-06 13:03:27 +02:00
parent 70a6c37a26
commit 17c7e05235
5 changed files with 213 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
ironic_python_agent/hardware.py
View File

@@ -21,6 +21,7 @@ import netifaces
from oslo_concurrency import processutils
from oslo_log import log
from oslo_utils import units
import pint
import psutil
import pyudev
import six
@@ -33,6 +34,10 @@ from ironic_python_agent import utils
_global_managers = None
LOG = log.getLogger()
UNIT_CONVERTER = pint.UnitRegistry(filename=None)
UNIT_CONVERTER.define('MB = []')
UNIT_CONVERTER.define('GB = 1024 MB')
class HardwareSupport(object):
"""Example priorities for hardware managers.
@@ -67,30 +72,34 @@ class BlockDevice(encoding.Serializable):
class NetworkInterface(encoding.Serializable):
serializable_fields = ('name', 'mac_address', 'switch_port_descr',
'switch_chassis_descr')
'switch_chassis_descr', 'ipv4_address')
def __init__(self, name, mac_addr):
def __init__(self, name, mac_addr, ipv4_address=None):
self.name = name
self.mac_address = mac_addr
self.ipv4_address = ipv4_address
# TODO(russellhaering): Pull these from LLDP
self.switch_port_descr = None
self.switch_chassis_descr = None
class CPU(encoding.Serializable):
serializable_fields = ('model_name', 'frequency', 'count')
serializable_fields = ('model_name', 'frequency', 'count', 'architecture')
def __init__(self, model_name, frequency, count):
def __init__(self, model_name, frequency, count, architecture):
self.model_name = model_name
self.frequency = frequency
self.count = count
self.architecture = architecture
class Memory(encoding.Serializable):
serializable_fields = ('total', )
serializable_fields = ('total', 'physical_mb')
# physical = total + kernel binary + reserved space
def __init__(self, total):
def __init__(self, total, physical_mb=None):
self.total = total
self.physical_mb = physical_mb
@six.add_metaclass(abc.ABCMeta)
@@ -114,6 +123,9 @@ class HardwareManager(object):
def get_os_install_device(self):
raise errors.IncompatibleHardwareMethodError
def get_bmc_address(self):
raise errors.IncompatibleHardwareMethodError()
def erase_block_device(self, node, block_device):
"""Attempt to erase a block device.
@@ -160,6 +172,7 @@ class HardwareManager(object):
hardware_info['cpu'] = self.get_cpus()
hardware_info['disks'] = self.list_block_devices()
hardware_info['memory'] = self.get_memory()
hardware_info['bmc_address'] = self.get_bmc_address()
return hardware_info
def get_clean_steps(self, node, ports):
@@ -242,11 +255,13 @@ class GenericHardwareManager(HardwareManager):
def _get_interface_info(self, interface_name):
addr_path = '{0}/class/net/{1}/address'.format(self.sys_path,
interface_name)
interface_name)
with open(addr_path) as addr_file:
mac_addr = addr_file.read().strip()
return NetworkInterface(interface_name, mac_addr)
return NetworkInterface(
interface_name, mac_addr,
ipv4_address=self.get_ipv4_addr(interface_name))
def get_ipv4_addr(self, interface_id):
try:
@@ -267,35 +282,53 @@ class GenericHardwareManager(HardwareManager):
for name in iface_names
if self._is_device(name)]
def _get_cpu_count(self):
if psutil.version_info[0] == 1:
return psutil.NUM_CPUS
elif psutil.version_info[0] == 2:
return psutil.cpu_count()
else:
raise AttributeError("Only psutil versions 1 and 2 supported")
def get_cpus(self):
model = None
freq = None
with open('/proc/cpuinfo') as f:
lines = f.read()
for line in lines.split('\n'):
if model and freq:
break
if not model and line.startswith('model name'):
model = line.split(':')[1].strip()
if not freq and line.startswith('cpu MHz'):
freq = line.split(':')[1].strip()
return CPU(model, freq, self._get_cpu_count())
lines = utils.execute('lscpu')[0]
cpu_info = {k.strip().lower(): v.strip() for k, v in
(line.split(':', 1)
for line in lines.split('\n')
if line.strip())}
# Current CPU frequency can be different from maximum one on modern
# processors
freq = cpu_info.get('cpu max mhz', cpu_info.get('cpu mhz'))
return CPU(model_name=cpu_info.get('model name'),
frequency=freq,
# this includes hyperthreading cores
count=int(cpu_info.get('cpu(s)')),
architecture=cpu_info.get('architecture'))
def get_memory(self):
# psutil returns a long, so we force it to an int
if psutil.version_info[0] == 1:
return Memory(int(psutil.TOTAL_PHYMEM))
total = int(psutil.TOTAL_PHYMEM)
elif psutil.version_info[0] == 2:
return Memory(int(psutil.phymem_usage().total))
total = int(psutil.phymem_usage().total)
try:
out, _e = utils.execute("dmidecode --type memory | grep Size",
shell=True)
except (processutils.ProcessExecutionError, OSError) as e:
LOG.warn("Cannot get real physical memory size: %s", e)
physical = None
else:
physical = 0
for line in out.strip().split('\n'):
line = line.strip()
if not line:
continue
try:
value = line.split(None, 1)[1].strip()
physical += int(UNIT_CONVERTER(value).to_base_units())
except Exception as exc:
LOG.error('Cannot parse size expression %s: %s',
line, exc)
if not physical:
LOG.warn('failed to get real physical RAM, dmidecode returned '
'%s', out)
return Memory(total=total, physical_mb=physical)
def list_block_devices(self):
"""List all physical block devices
@@ -538,6 +571,23 @@ class GenericHardwareManager(HardwareManager):
return True
def get_bmc_address(self):
# These modules are rarely loaded automatically
utils.try_execute('modprobe', 'ipmi_msghandler')
utils.try_execute('modprobe', 'ipmi_devintf')
utils.try_execute('modprobe', 'ipmi_si')
try:
out, _e = utils.execute(
"ipmitool lan print | grep -e 'IP Address [^S]' "
"| awk '{ print $4 }'", shell=True)
except (processutils.ProcessExecutionError, OSError) as e:
# Not error, because it's normal in virtual environment
LOG.warn("Cannot get BMC address: %s", e)
return
return out.strip()
def _compare_extensions(ext1, ext2):
mgr1 = ext1.obj

184
ironic_python_agent/tests/unit/test_hardware.py
View File

@@ -13,6 +13,7 @@
# limitations under the License.
import mock
import netifaces
import os
from oslo_concurrency import processutils
from oslotest import base as test_base
@@ -121,7 +122,7 @@ HDPARM_INFO_TEMPLATE = (
BLK_DEVICE_TEMPLATE = (
'KNAME="sda" MODEL="TinyUSB Drive" SIZE="3116853504" '
'ROTA="0" TYPE="disk"\n'
'ROTA="0" TYPE="disk" SERIAL="123"\n'
'KNAME="sdb" MODEL="Fastable SD131 7" SIZE="10737418240" '
'ROTA="0" TYPE="disk"\n'
'KNAME="sdc" MODEL="NWD-BLP4-1600 " SIZE="1765517033472" '
@@ -145,6 +146,59 @@ SHRED_OUTPUT = (
)
LSCPU_OUTPUT = """
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 4
On-line CPU(s) list: 0-3
Thread(s) per core: 1
Core(s) per socket: 4
Socket(s): 1
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 45
Model name: Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz
Stepping: 7
CPU MHz: 1290.000
CPU max MHz: 2400.0000
CPU min MHz: 1200.0000
BogoMIPS: 4800.06
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 10240K
NUMA node0 CPU(s): 0-3
"""
LSCPU_OUTPUT_NO_MAX_MHZ = """
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 12
On-line CPU(s) list: 0-11
Thread(s) per core: 2
Core(s) per socket: 6
Socket(s): 1
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 63
Model name: Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz
Stepping: 2
CPU MHz: 1794.433
BogoMIPS: 6983.57
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 15360K
NUMA node0 CPU(s): 0-11
"""
class FakeHardwareManager(hardware.GenericHardwareManager):
def __init__(self, hardware_support):
self._hardware_support = hardware_support
@@ -185,30 +239,37 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
self.node = {'uuid': 'dda135fb-732d-4742-8e72-df8f3199d244',
'driver_internal_info': {}}
@mock.patch('netifaces.ifaddresses')
@mock.patch('os.listdir')
@mock.patch('os.path.exists')
@mock.patch(OPEN_FUNCTION_NAME)
def test_list_network_interfaces(self,
mocked_open,
mocked_exists,
mocked_listdir):
mocked_listdir,
mocked_ifaddresses):
mocked_listdir.return_value = ['lo', 'eth0']
mocked_exists.side_effect = [False, True]
mocked_open.return_value.__enter__ = lambda s: s
mocked_open.return_value.__exit__ = mock.Mock()
read_mock = mocked_open.return_value.read
read_mock.return_value = '00:0c:29:8c:11:b1\n'
mocked_ifaddresses.return_value = {
netifaces.AF_INET: [{'addr': '192.168.1.2'}]
}
interfaces = self.hardware.list_network_interfaces()
self.assertEqual(len(interfaces), 1)
self.assertEqual(interfaces[0].name, 'eth0')
self.assertEqual(interfaces[0].mac_address, '00:0c:29:8c:11:b1')
self.assertEqual(interfaces[0].ipv4_address, '192.168.1.2')
@mock.patch.object(utils, 'execute')
def test_get_os_install_device(self, mocked_execute):
mocked_execute.return_value = (BLK_DEVICE_TEMPLATE, '')
self.assertEqual(self.hardware.get_os_install_device(), '/dev/sdb')
mocked_execute.assert_called_once_with(
'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE', check_exit_code=[0])
'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE',
check_exit_code=[0])
@mock.patch.object(hardware.GenericHardwareManager, '_get_device_vendor')
@mock.patch.object(pyudev.Device, 'from_device_file')
@@ -232,7 +293,8 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
self.assertEqual('/dev/sdb', self.hardware.get_os_install_device())
mocked_execute.assert_called_once_with(
'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE', check_exit_code=[0])
'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE',
check_exit_code=[0])
mock_root_device.assert_called_once_with()
expected = [mock.call(mock.ANY, '/dev/sda'),
mock.call(mock.ANY, '/dev/sdb')]
@@ -249,7 +311,8 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
self.assertRaises(errors.DeviceNotFound,
self.hardware.get_os_install_device)
mocked_execute.assert_called_once_with(
'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE', check_exit_code=[0])
'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE',
check_exit_code=[0])
mock_root_device.assert_called_once_with()
expected = [mock.call(mock.ANY, '/dev/sda'),
mock.call(mock.ANY, '/dev/sdb'),
@@ -265,74 +328,42 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
'/sys/class/block/sdfake/device/vendor', 'r')
self.assertEqual('fake-vendor', vendor)
@mock.patch('ironic_python_agent.hardware.GenericHardwareManager.'
'_get_cpu_count')
@mock.patch(OPEN_FUNCTION_NAME)
def test_get_cpus(self, mocked_open, mocked_cpucount):
mocked_open.return_value.__enter__ = lambda s: s
mocked_open.return_value.__exit__ = mock.Mock()
read_mock = mocked_open.return_value.read
read_mock.return_value = (
'processor : 0\n'
'vendor_id : GenuineIntel\n'
'cpu family : 6\n'
'model : 58\n'
'model name : Intel(R) Core(TM) i7-3720QM CPU @ 2.60GHz\n'
'stepping : 9\n'
'microcode : 0x15\n'
'cpu MHz : 2594.685\n'
'cache size : 6144 KB\n'
'fpu : yes\n'
'fpu_exception : yes\n'
'cpuid level : 13\n'
'wp : yes\n'
'flags : fpu vme de pse tsc msr pae mce cx8 apic sep '
'mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss '
'syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl '
'xtopology tsc_reliable nonstop_tsc aperfmperf eagerfpu pni '
'pclmulqdq ssse3 cx16 pcid sse4_1 sse4_2 x2apic popcnt aes xsave '
'avx f16c rdrand hypervisor lahf_lm ida arat epb xsaveopt pln pts '
'dtherm fsgsbase smep\n'
'bogomips : 5189.37\n'
'clflush size : 64\n'
'cache_alignment : 64\n'
'address sizes : 40 bits physical, 48 bits virtual\n'
'power management:\n'
'\n'
'processor : 1\n'
'vendor_id : GenuineIntel\n'
'cpu family : 6\n'
'model : 58\n'
'model name : Intel(R) Core(TM) i7-3720QM CPU @ 2.60GHz\n'
'stepping : 9\n'
'microcode : 0x15\n'
'cpu MHz : 2594.685\n'
'cache size : 6144 KB\n'
'fpu : yes\n'
'fpu_exception : yes\n'
'cpuid level : 13\n'
'wp : yes\n'
'flags : fpu vme de pse tsc msr pae mce cx8 apic sep '
'mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss '
'syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl '
'xtopology tsc_reliable nonstop_tsc aperfmperf eagerfpu pni '
'pclmulqdq ssse3 cx16 pcid sse4_1 sse4_2 x2apic popcnt aes xsave '
'avx f16c rdrand hypervisor lahf_lm ida arat epb xsaveopt pln pts '
'dtherm fsgsbase smep\n'
'bogomips : 5189.37\n'
'clflush size : 64\n'
'cache_alignment : 64\n'
'address sizes : 40 bits physical, 48 bits virtual\n'
'power management:\n'
)
mocked_cpucount.return_value = 2
@mock.patch.object(utils, 'execute')
def test_get_cpus(self, mocked_execute):
mocked_execute.return_value = LSCPU_OUTPUT, ''
cpus = self.hardware.get_cpus()
self.assertEqual(cpus.model_name,
'Intel(R) Core(TM) i7-3720QM CPU @ 2.60GHz')
self.assertEqual(cpus.frequency, '2594.685')
self.assertEqual(cpus.count, 2)
'Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz')
self.assertEqual(cpus.frequency, '2400.0000')
self.assertEqual(cpus.count, 4)
self.assertEqual(cpus.architecture, 'x86_64')
@mock.patch.object(utils, 'execute')
def test_get_cpus2(self, mocked_execute):
mocked_execute.return_value = LSCPU_OUTPUT_NO_MAX_MHZ, ''
cpus = self.hardware.get_cpus()
self.assertEqual(cpus.model_name,
'Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz')
self.assertEqual(cpus.frequency, '1794.433')
self.assertEqual(cpus.count, 12)
self.assertEqual(cpus.architecture, 'x86_64')
@mock.patch('psutil.version_info', (2, 0))
@mock.patch('psutil.phymem_usage', autospec=True)
@mock.patch.object(utils, 'execute')
def test_get_memory(self, mocked_execute, mocked_usage):
mocked_usage.return_value = mock.Mock(total=3952 * 1024 * 1024)
mocked_execute.return_value = (
"Foo\nSize: 2048 MB\nSize: 2 GB\n",
""
)
mem = self.hardware.get_memory()
self.assertEqual(mem.total, 3952 * 1024 * 1024)
self.assertEqual(mem.physical_mb, 4096)
def test_list_hardware_info(self):
self.hardware.list_network_interfaces = mock.Mock()
@@ -345,7 +376,8 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
self.hardware.get_cpus.return_value = hardware.CPU(
'Awesome CPU x14 9001',
9001,
14)
14,
'x86_64')
self.hardware.get_memory = mock.Mock()
self.hardware.get_memory.return_value = hardware.Memory(1017012)
@@ -630,3 +662,13 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
'\tsupported: enhanced erase', '--security-erase-enhanced')
test_security_erase_option(self,
'\tnot\tsupported: enhanced erase', '--security-erase')
@mock.patch.object(utils, 'execute')
def test_get_bmc_address(self, mocked_execute):
mocked_execute.return_value = '192.1.2.3\n', ''
self.assertEqual('192.1.2.3', self.hardware.get_bmc_address())
@mock.patch.object(utils, 'execute')
def test_get_bmc_address_virt(self, mocked_execute):
mocked_execute.side_effect = processutils.ProcessExecutionError()
self.assertIsNone(self.hardware.get_bmc_address())

14
ironic_python_agent/tests/unit/test_ironic_api_client.py
View File

@@ -44,12 +44,14 @@ class TestBaseIronicPythonAgent(test_base.BaseTestCase):
hardware.NetworkInterface('eth0', '00:0c:29:8c:11:b1'),
hardware.NetworkInterface('eth1', '00:0c:29:8c:11:b2'),
],
'cpu': hardware.CPU('Awesome Jay CPU x10 9001', '9001', '10'),
'cpu': hardware.CPU('Awesome Jay CPU x10 9001', '9001', '10',
'ARMv9'),
'disks': [
hardware.BlockDevice('/dev/sdj', 'small', '9001', False),
hardware.BlockDevice('/dev/hdj', 'big', '9002', False),
],
'memory': hardware.Memory('8675309'),
'memory': hardware.Memory(total='8675309',
physical_mb='8675'),
}
def test_successful_heartbeat(self):
@@ -145,12 +147,14 @@ class TestBaseIronicPythonAgent(test_base.BaseTestCase):
{
u'mac_address': u'00:0c:29:8c:11:b1',
u'name': u'eth0',
u'ipv4_address': None,
u'switch_chassis_descr': None,
u'switch_port_descr': None
},
{
u'mac_address': u'00:0c:29:8c:11:b2',
u'name': u'eth1',
u'ipv4_address': None,
u'switch_chassis_descr': None,
'switch_port_descr': None
}
@@ -159,23 +163,25 @@ class TestBaseIronicPythonAgent(test_base.BaseTestCase):
u'model_name': u'Awesome Jay CPU x10 9001',
u'frequency': u'9001',
u'count': u'10',
u'architecture': u'ARMv9'
},
u'disks': [
{
u'model': u'small',
u'name': u'/dev/sdj',
u'rotational': False,
u'size': u'9001'
u'size': u'9001',
},
{
u'model': u'big',
u'name': u'/dev/hdj',
u'rotational': False,
u'size': u'9002'
u'size': u'9002',
}
],
u'memory': {
u'total': u'8675309',
u'physical_mb': u'8675'
},
})

8
ironic_python_agent/utils.py
View File

@@ -64,6 +64,14 @@ def execute(*cmd, **kwargs):
return result
def try_execute(*cmd, **kwargs):
"""The same as execute but returns None on error."""
try:
return execute(*cmd, **kwargs)
except (processutils.ProcessExecutionError, OSError) as e:
LOG.debug('Command failed: %s', e)
def _read_params_from_file(filepath):
"""Extract key=value pairs from a file.

1
requirements.txt
View File

@@ -15,6 +15,7 @@ oslo.serialization>=1.4.0 # Apache-2.0
oslo.service>=0.6.0 # Apache-2.0
oslo.utils>=2.0.0 # Apache-2.0
pecan>=1.0.0
Pint>=0.5 # BSD
psutil<2.0.0,>=1.1.1
pyudev
requests>=2.5.2