Extend hardware manager with data needed for inspector

* Added NetworkInterface.ip4_address
* Added HardwareManager.get_bmc_address()
* Added Memory.physical_mb

  This is total memory as reported by dmidecode, and yes,
  it's different from total, as it includes kernel reserved space.

* Added CPU.architecture

  As a side effect, get_cpus was switched to lscpu.
  Also fixes problem when get_cpus reported the current frequency
  instead of maximum one.

Change-Id: I4080d4d551eb0bb995a94ef9a300351910c09fb9

ironic_python_agent/hardware.py

@@ -21,6 +21,7 @@ import netifaces
 from oslo_concurrency import processutils
 from oslo_log import log
 from oslo_utils import units
+import pint
 import psutil
 import pyudev
 import six
@@ -33,6 +34,10 @@ from ironic_python_agent import utils
 _global_managers = None
 LOG = log.getLogger()
 
+UNIT_CONVERTER = pint.UnitRegistry(filename=None)
+UNIT_CONVERTER.define('MB = []')
+UNIT_CONVERTER.define('GB = 1024 MB')
+
 
 class HardwareSupport(object):
     """Example priorities for hardware managers.
@@ -67,30 +72,34 @@ class BlockDevice(encoding.Serializable):
 
 class NetworkInterface(encoding.Serializable):
     serializable_fields = ('name', 'mac_address', 'switch_port_descr',
-                           'switch_chassis_descr')
+                           'switch_chassis_descr', 'ipv4_address')
 
-    def __init__(self, name, mac_addr):
+    def __init__(self, name, mac_addr, ipv4_address=None):
         self.name = name
         self.mac_address = mac_addr
+        self.ipv4_address = ipv4_address
         # TODO(russellhaering): Pull these from LLDP
         self.switch_port_descr = None
         self.switch_chassis_descr = None
 
 
 class CPU(encoding.Serializable):
-    serializable_fields = ('model_name', 'frequency', 'count')
+    serializable_fields = ('model_name', 'frequency', 'count', 'architecture')
 
-    def __init__(self, model_name, frequency, count):
+    def __init__(self, model_name, frequency, count, architecture):
         self.model_name = model_name
         self.frequency = frequency
         self.count = count
+        self.architecture = architecture
 
 
 class Memory(encoding.Serializable):
-    serializable_fields = ('total', )
+    serializable_fields = ('total', 'physical_mb')
+    # physical = total + kernel binary + reserved space
 
-    def __init__(self, total):
+    def __init__(self, total, physical_mb=None):
         self.total = total
+        self.physical_mb = physical_mb
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -114,6 +123,9 @@ class HardwareManager(object):
     def get_os_install_device(self):
         raise errors.IncompatibleHardwareMethodError
 
+    def get_bmc_address(self):
+        raise errors.IncompatibleHardwareMethodError()
+
     def erase_block_device(self, node, block_device):
         """Attempt to erase a block device.
 
@@ -160,6 +172,7 @@ class HardwareManager(object):
         hardware_info['cpu'] = self.get_cpus()
         hardware_info['disks'] = self.list_block_devices()
         hardware_info['memory'] = self.get_memory()
+        hardware_info['bmc_address'] = self.get_bmc_address()
         return hardware_info
 
     def get_clean_steps(self, node, ports):
@@ -246,7 +259,9 @@ class GenericHardwareManager(HardwareManager):
         with open(addr_path) as addr_file:
             mac_addr = addr_file.read().strip()
 
-        return NetworkInterface(interface_name, mac_addr)
+        return NetworkInterface(
+            interface_name, mac_addr,
+            ipv4_address=self.get_ipv4_addr(interface_name))
 
     def get_ipv4_addr(self, interface_id):
         try:
@@ -267,35 +282,53 @@ class GenericHardwareManager(HardwareManager):
                 for name in iface_names
                 if self._is_device(name)]
 
-    def _get_cpu_count(self):
-        if psutil.version_info[0] == 1:
-            return psutil.NUM_CPUS
-        elif psutil.version_info[0] == 2:
-            return psutil.cpu_count()
-        else:
-            raise AttributeError("Only psutil versions 1 and 2 supported")
-
     def get_cpus(self):
-        model = None
+        lines = utils.execute('lscpu')[0]
-        freq = None
+        cpu_info = {k.strip().lower(): v.strip() for k, v in
-        with open('/proc/cpuinfo') as f:
+                    (line.split(':', 1)
-            lines = f.read()
+                     for line in lines.split('\n')
-            for line in lines.split('\n'):
+                     if line.strip())}
-                if model and freq:
+        # Current CPU frequency can be different from maximum one on modern
-                    break
+        # processors
-                if not model and line.startswith('model name'):
+        freq = cpu_info.get('cpu max mhz', cpu_info.get('cpu mhz'))
-                    model = line.split(':')[1].strip()
+        return CPU(model_name=cpu_info.get('model name'),
-                if not freq and line.startswith('cpu MHz'):
+                   frequency=freq,
-                    freq = line.split(':')[1].strip()
+                   # this includes hyperthreading cores
-
+                   count=int(cpu_info.get('cpu(s)')),
-        return CPU(model, freq, self._get_cpu_count())
+                   architecture=cpu_info.get('architecture'))
 
     def get_memory(self):
         # psutil returns a long, so we force it to an int
         if psutil.version_info[0] == 1:
-            return Memory(int(psutil.TOTAL_PHYMEM))
+            total = int(psutil.TOTAL_PHYMEM)
         elif psutil.version_info[0] == 2:
-            return Memory(int(psutil.phymem_usage().total))
+            total = int(psutil.phymem_usage().total)
+
+        try:
+            out, _e = utils.execute("dmidecode --type memory | grep Size",
+                                    shell=True)
+        except (processutils.ProcessExecutionError, OSError) as e:
+            LOG.warn("Cannot get real physical memory size: %s", e)
+            physical = None
+        else:
+            physical = 0
+            for line in out.strip().split('\n'):
+                line = line.strip()
+                if not line:
+                    continue
+
+                try:
+                    value = line.split(None, 1)[1].strip()
+                    physical += int(UNIT_CONVERTER(value).to_base_units())
+                except Exception as exc:
+                    LOG.error('Cannot parse size expression %s: %s',
+                              line, exc)
+
+            if not physical:
+                LOG.warn('failed to get real physical RAM, dmidecode returned '
+                         '%s', out)
+
+        return Memory(total=total, physical_mb=physical)
 
     def list_block_devices(self):
         """List all physical block devices
@@ -538,6 +571,23 @@ class GenericHardwareManager(HardwareManager):
 
         return True
 
+    def get_bmc_address(self):
+        # These modules are rarely loaded automatically
+        utils.try_execute('modprobe', 'ipmi_msghandler')
+        utils.try_execute('modprobe', 'ipmi_devintf')
+        utils.try_execute('modprobe', 'ipmi_si')
+
+        try:
+            out, _e = utils.execute(
+                "ipmitool lan print | grep -e 'IP Address [^S]' "
+                "| awk '{ print $4 }'", shell=True)
+        except (processutils.ProcessExecutionError, OSError) as e:
+            # Not error, because it's normal in virtual environment
+            LOG.warn("Cannot get BMC address: %s", e)
+            return
+
+        return out.strip()
+
 
 def _compare_extensions(ext1, ext2):
     mgr1 = ext1.obj

ironic_python_agent/tests/unit/test_hardware.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import mock
+import netifaces
 import os
 from oslo_concurrency import processutils
 from oslotest import base as test_base
@@ -121,7 +122,7 @@ HDPARM_INFO_TEMPLATE = (
 
 BLK_DEVICE_TEMPLATE = (
     'KNAME="sda" MODEL="TinyUSB Drive" SIZE="3116853504" '
-    'ROTA="0" TYPE="disk"\n'
+    'ROTA="0" TYPE="disk" SERIAL="123"\n'
     'KNAME="sdb" MODEL="Fastable SD131 7" SIZE="10737418240" '
     'ROTA="0" TYPE="disk"\n'
     'KNAME="sdc" MODEL="NWD-BLP4-1600   " SIZE="1765517033472" '
@@ -145,6 +146,59 @@ SHRED_OUTPUT = (
 )
 
 
+LSCPU_OUTPUT = """
+Architecture:          x86_64
+CPU op-mode(s):        32-bit, 64-bit
+Byte Order:            Little Endian
+CPU(s):                4
+On-line CPU(s) list:   0-3
+Thread(s) per core:    1
+Core(s) per socket:    4
+Socket(s):             1
+NUMA node(s):          1
+Vendor ID:             GenuineIntel
+CPU family:            6
+Model:                 45
+Model name:            Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz
+Stepping:              7
+CPU MHz:               1290.000
+CPU max MHz:           2400.0000
+CPU min MHz:           1200.0000
+BogoMIPS:              4800.06
+Virtualization:        VT-x
+L1d cache:             32K
+L1i cache:             32K
+L2 cache:              256K
+L3 cache:              10240K
+NUMA node0 CPU(s):     0-3
+"""
+
+LSCPU_OUTPUT_NO_MAX_MHZ = """
+Architecture:          x86_64
+CPU op-mode(s):        32-bit, 64-bit
+Byte Order:            Little Endian
+CPU(s):                12
+On-line CPU(s) list:   0-11
+Thread(s) per core:    2
+Core(s) per socket:    6
+Socket(s):             1
+NUMA node(s):          1
+Vendor ID:             GenuineIntel
+CPU family:            6
+Model:                 63
+Model name:            Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz
+Stepping:              2
+CPU MHz:               1794.433
+BogoMIPS:              6983.57
+Virtualization:        VT-x
+L1d cache:             32K
+L1i cache:             32K
+L2 cache:              256K
+L3 cache:              15360K
+NUMA node0 CPU(s):     0-11
+"""
+
+
 class FakeHardwareManager(hardware.GenericHardwareManager):
     def __init__(self, hardware_support):
         self._hardware_support = hardware_support
@@ -185,30 +239,37 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
         self.node = {'uuid': 'dda135fb-732d-4742-8e72-df8f3199d244',
                      'driver_internal_info': {}}
 
+    @mock.patch('netifaces.ifaddresses')
     @mock.patch('os.listdir')
     @mock.patch('os.path.exists')
     @mock.patch(OPEN_FUNCTION_NAME)
     def test_list_network_interfaces(self,
                                      mocked_open,
                                      mocked_exists,
-                                     mocked_listdir):
+                                     mocked_listdir,
+                                     mocked_ifaddresses):
         mocked_listdir.return_value = ['lo', 'eth0']
         mocked_exists.side_effect = [False, True]
         mocked_open.return_value.__enter__ = lambda s: s
         mocked_open.return_value.__exit__ = mock.Mock()
         read_mock = mocked_open.return_value.read
         read_mock.return_value = '00:0c:29:8c:11:b1\n'
+        mocked_ifaddresses.return_value = {
+            netifaces.AF_INET: [{'addr': '192.168.1.2'}]
+        }
         interfaces = self.hardware.list_network_interfaces()
         self.assertEqual(len(interfaces), 1)
         self.assertEqual(interfaces[0].name, 'eth0')
         self.assertEqual(interfaces[0].mac_address, '00:0c:29:8c:11:b1')
+        self.assertEqual(interfaces[0].ipv4_address, '192.168.1.2')
 
     @mock.patch.object(utils, 'execute')
     def test_get_os_install_device(self, mocked_execute):
         mocked_execute.return_value = (BLK_DEVICE_TEMPLATE, '')
         self.assertEqual(self.hardware.get_os_install_device(), '/dev/sdb')
         mocked_execute.assert_called_once_with(
-            'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE', check_exit_code=[0])
+            'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE',
+            check_exit_code=[0])
 
     @mock.patch.object(hardware.GenericHardwareManager, '_get_device_vendor')
     @mock.patch.object(pyudev.Device, 'from_device_file')
@@ -232,7 +293,8 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
 
         self.assertEqual('/dev/sdb', self.hardware.get_os_install_device())
         mocked_execute.assert_called_once_with(
-            'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE', check_exit_code=[0])
+            'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE',
+            check_exit_code=[0])
         mock_root_device.assert_called_once_with()
         expected = [mock.call(mock.ANY, '/dev/sda'),
                     mock.call(mock.ANY, '/dev/sdb')]
@@ -249,7 +311,8 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
         self.assertRaises(errors.DeviceNotFound,
                           self.hardware.get_os_install_device)
         mocked_execute.assert_called_once_with(
-            'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE', check_exit_code=[0])
+            'lsblk', '-PbdioKNAME,MODEL,SIZE,ROTA,TYPE',
+            check_exit_code=[0])
         mock_root_device.assert_called_once_with()
         expected = [mock.call(mock.ANY, '/dev/sda'),
                     mock.call(mock.ANY, '/dev/sdb'),
@@ -265,74 +328,42 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
                 '/sys/class/block/sdfake/device/vendor', 'r')
             self.assertEqual('fake-vendor', vendor)
 
-    @mock.patch('ironic_python_agent.hardware.GenericHardwareManager.'
+    @mock.patch.object(utils, 'execute')
-                '_get_cpu_count')
+    def test_get_cpus(self, mocked_execute):
-    @mock.patch(OPEN_FUNCTION_NAME)
+        mocked_execute.return_value = LSCPU_OUTPUT, ''
-    def test_get_cpus(self, mocked_open, mocked_cpucount):
-        mocked_open.return_value.__enter__ = lambda s: s
-        mocked_open.return_value.__exit__ = mock.Mock()
-        read_mock = mocked_open.return_value.read
-        read_mock.return_value = (
-            'processor       : 0\n'
-            'vendor_id       : GenuineIntel\n'
-            'cpu family      : 6\n'
-            'model           : 58\n'
-            'model name      : Intel(R) Core(TM) i7-3720QM CPU @ 2.60GHz\n'
-            'stepping        : 9\n'
-            'microcode       : 0x15\n'
-            'cpu MHz         : 2594.685\n'
-            'cache size      : 6144 KB\n'
-            'fpu             : yes\n'
-            'fpu_exception   : yes\n'
-            'cpuid level     : 13\n'
-            'wp              : yes\n'
-            'flags           : fpu vme de pse tsc msr pae mce cx8 apic sep '
-            'mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss '
-            'syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl '
-            'xtopology tsc_reliable nonstop_tsc aperfmperf eagerfpu pni '
-            'pclmulqdq ssse3 cx16 pcid sse4_1 sse4_2 x2apic popcnt aes xsave '
-            'avx f16c rdrand hypervisor lahf_lm ida arat epb xsaveopt pln pts '
-            'dtherm fsgsbase smep\n'
-            'bogomips        : 5189.37\n'
-            'clflush size    : 64\n'
-            'cache_alignment : 64\n'
-            'address sizes   : 40 bits physical, 48 bits virtual\n'
-            'power management:\n'
-            '\n'
-            'processor       : 1\n'
-            'vendor_id       : GenuineIntel\n'
-            'cpu family      : 6\n'
-            'model           : 58\n'
-            'model name      : Intel(R) Core(TM) i7-3720QM CPU @ 2.60GHz\n'
-            'stepping        : 9\n'
-            'microcode       : 0x15\n'
-            'cpu MHz         : 2594.685\n'
-            'cache size      : 6144 KB\n'
-            'fpu             : yes\n'
-            'fpu_exception   : yes\n'
-            'cpuid level     : 13\n'
-            'wp              : yes\n'
-            'flags           : fpu vme de pse tsc msr pae mce cx8 apic sep '
-            'mtrr pge mca cmov pat pse36 clflush dts mmx fxsr sse sse2 ss '
-            'syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl '
-            'xtopology tsc_reliable nonstop_tsc aperfmperf eagerfpu pni '
-            'pclmulqdq ssse3 cx16 pcid sse4_1 sse4_2 x2apic popcnt aes xsave '
-            'avx f16c rdrand hypervisor lahf_lm ida arat epb xsaveopt pln pts '
-            'dtherm fsgsbase smep\n'
-            'bogomips        : 5189.37\n'
-            'clflush size    : 64\n'
-            'cache_alignment : 64\n'
-            'address sizes   : 40 bits physical, 48 bits virtual\n'
-            'power management:\n'
-        )
-
-        mocked_cpucount.return_value = 2
 
         cpus = self.hardware.get_cpus()
         self.assertEqual(cpus.model_name,
-                         'Intel(R) Core(TM) i7-3720QM CPU @ 2.60GHz')
+                         'Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz')
-        self.assertEqual(cpus.frequency, '2594.685')
+        self.assertEqual(cpus.frequency, '2400.0000')
-        self.assertEqual(cpus.count, 2)
+        self.assertEqual(cpus.count, 4)
+        self.assertEqual(cpus.architecture, 'x86_64')
+
+    @mock.patch.object(utils, 'execute')
+    def test_get_cpus2(self, mocked_execute):
+        mocked_execute.return_value = LSCPU_OUTPUT_NO_MAX_MHZ, ''
+
+        cpus = self.hardware.get_cpus()
+        self.assertEqual(cpus.model_name,
+                         'Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz')
+        self.assertEqual(cpus.frequency, '1794.433')
+        self.assertEqual(cpus.count, 12)
+        self.assertEqual(cpus.architecture, 'x86_64')
+
+    @mock.patch('psutil.version_info', (2, 0))
+    @mock.patch('psutil.phymem_usage', autospec=True)
+    @mock.patch.object(utils, 'execute')
+    def test_get_memory(self, mocked_execute, mocked_usage):
+        mocked_usage.return_value = mock.Mock(total=3952 * 1024 * 1024)
+        mocked_execute.return_value = (
+            "Foo\nSize: 2048 MB\nSize: 2 GB\n",
+            ""
+        )
+
+        mem = self.hardware.get_memory()
+
+        self.assertEqual(mem.total, 3952 * 1024 * 1024)
+        self.assertEqual(mem.physical_mb, 4096)
 
     def test_list_hardware_info(self):
         self.hardware.list_network_interfaces = mock.Mock()
@@ -345,7 +376,8 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
         self.hardware.get_cpus.return_value = hardware.CPU(
             'Awesome CPU x14 9001',
             9001,
-            14)
+            14,
+            'x86_64')
 
         self.hardware.get_memory = mock.Mock()
         self.hardware.get_memory.return_value = hardware.Memory(1017012)
@@ -630,3 +662,13 @@ class TestGenericHardwareManager(test_base.BaseTestCase):
                 '\tsupported: enhanced erase', '--security-erase-enhanced')
         test_security_erase_option(self,
                 '\tnot\tsupported: enhanced erase', '--security-erase')
+
+    @mock.patch.object(utils, 'execute')
+    def test_get_bmc_address(self, mocked_execute):
+        mocked_execute.return_value = '192.1.2.3\n', ''
+        self.assertEqual('192.1.2.3', self.hardware.get_bmc_address())
+
+    @mock.patch.object(utils, 'execute')
+    def test_get_bmc_address_virt(self, mocked_execute):
+        mocked_execute.side_effect = processutils.ProcessExecutionError()
+        self.assertIsNone(self.hardware.get_bmc_address())

ironic_python_agent/tests/unit/test_ironic_api_client.py

@@ -44,12 +44,14 @@ class TestBaseIronicPythonAgent(test_base.BaseTestCase):
                 hardware.NetworkInterface('eth0', '00:0c:29:8c:11:b1'),
                 hardware.NetworkInterface('eth1', '00:0c:29:8c:11:b2'),
             ],
-            'cpu': hardware.CPU('Awesome Jay CPU x10 9001', '9001', '10'),
+            'cpu': hardware.CPU('Awesome Jay CPU x10 9001', '9001', '10',
+                                'ARMv9'),
             'disks': [
                 hardware.BlockDevice('/dev/sdj', 'small', '9001', False),
                 hardware.BlockDevice('/dev/hdj', 'big', '9002', False),
             ],
-            'memory': hardware.Memory('8675309'),
+            'memory': hardware.Memory(total='8675309',
+                                      physical_mb='8675'),
         }
 
     def test_successful_heartbeat(self):
@@ -145,12 +147,14 @@ class TestBaseIronicPythonAgent(test_base.BaseTestCase):
                 {
                     u'mac_address': u'00:0c:29:8c:11:b1',
                     u'name': u'eth0',
+                    u'ipv4_address': None,
                     u'switch_chassis_descr': None,
                     u'switch_port_descr': None
                 },
                 {
                     u'mac_address': u'00:0c:29:8c:11:b2',
                     u'name': u'eth1',
+                    u'ipv4_address': None,
                     u'switch_chassis_descr': None,
                     'switch_port_descr': None
                 }
@@ -159,23 +163,25 @@ class TestBaseIronicPythonAgent(test_base.BaseTestCase):
                 u'model_name': u'Awesome Jay CPU x10 9001',
                 u'frequency': u'9001',
                 u'count': u'10',
+                u'architecture': u'ARMv9'
             },
             u'disks': [
                 {
                     u'model': u'small',
                     u'name': u'/dev/sdj',
                     u'rotational': False,
-                    u'size': u'9001'
+                    u'size': u'9001',
                 },
                 {
                     u'model': u'big',
                     u'name': u'/dev/hdj',
                     u'rotational': False,
-                    u'size': u'9002'
+                    u'size': u'9002',
                 }
             ],
             u'memory': {
                 u'total': u'8675309',
+                u'physical_mb': u'8675'
             },
         })
 

ironic_python_agent/utils.py

@@ -64,6 +64,14 @@ def execute(*cmd, **kwargs):
     return result
 
 
+def try_execute(*cmd, **kwargs):
+    """The same as execute but returns None on error."""
+    try:
+        return execute(*cmd, **kwargs)
+    except (processutils.ProcessExecutionError, OSError) as e:
+        LOG.debug('Command failed: %s', e)
+
+
 def _read_params_from_file(filepath):
     """Extract key=value pairs from a file.
 

requirements.txt

@@ -15,6 +15,7 @@ oslo.serialization>=1.4.0 # Apache-2.0
 oslo.service>=0.6.0 # Apache-2.0
 oslo.utils>=2.0.0 # Apache-2.0
 pecan>=1.0.0
+Pint>=0.5 # BSD
 psutil<2.0.0,>=1.1.1
 pyudev
 requests>=2.5.2