Ironic python agent does not extract correct available memory

This patch fixes parsing of memory class output of lshw output.
Task: 30201
Story: 2005308

Change-Id: I0e7ff9f86a266d2f798457136fed0100c78569e2

ironic_python_agent/hardware.py

@@ -18,6 +18,7 @@ import functools
 import json
 from multiprocessing.pool import ThreadPool
 import os
+import re
 import shlex
 import time
 
@@ -47,7 +48,7 @@ WARN_BIOSDEVNAME_NOT_FOUND = False
 UNIT_CONVERTER = pint.UnitRegistry(filename=None)
 UNIT_CONVERTER.define('bytes = []')
 UNIT_CONVERTER.define('MB = 1048576 bytes')
-
+_MEMORY_ID_RE = re.compile(r'^memory(:\d+)?$')
 NODE = None
 
 
@@ -759,11 +760,13 @@ class GenericHardwareManager(HardwareManager):
             for sys_child in sys_dict['children']:
                 if sys_child['id'] == 'core':
                     for core_child in sys_child['children']:
-                        if core_child['id'] == 'memory':
+                        if _MEMORY_ID_RE.match(core_child['id']):
-                            if core_child.get('size'):
+                            for bank in core_child.get('children', ()):
-                                value = "%(size)s %(units)s" % core_child
+                                if bank.get('size'):
-                                physical += int(UNIT_CONVERTER(value).to(
+                                    value = ("%(size)s %(units)s" % bank)
-                                    'MB').magnitude)
+                                    physical += int(UNIT_CONVERTER(value).to
+                                                    ('MB').magnitude)
+
             if not physical:
                 LOG.warning('Did not find any physical RAM')
 

ironic_python_agent/tests/unit/test_hardware.py

@@ -268,7 +268,7 @@ CPUINFO_FLAGS_OUTPUT = """
 flags           : fpu vme de pse
 """
 
-LSHW_JSON_OUTPUT = ("""
+LSHW_JSON_OUTPUT_V1 = ("""
 {
   "id": "fuzzypickles",
   "product": "ABC123 (GENERIC_SERVER)",
@@ -377,6 +377,194 @@ LSHW_JSON_OUTPUT = ("""
 }
 """, "")
 
+LSHW_JSON_OUTPUT_V2 = ("""
+{
+  "id" : "bumblebee",
+  "class" : "system",
+  "claimed" : true,
+  "handle" : "DMI:0001",
+  "description" : "Rack Mount Chassis",
+  "product" : "ABCD",
+  "vendor" : "ABCD",
+  "version" : "1234",
+  "serial" : "1234",
+  "width" : 64,
+  "configuration" : {
+    "boot" : "normal",
+    "chassis" : "rackmount",
+    "family" : "Intel Grantley EP",
+    "sku" : "NULL",
+    "uuid" : "00010002-0003-0004-0005-000600070008"
+  },
+  "capabilities" : {
+    "smbios-2.8" : "SMBIOS version 2.8",
+    "dmi-2.7" : "DMI version 2.7",
+    "vsyscall32" : "32-bit processes"
+  },
+  "children" : [
+    {
+      "id" : "core",
+      "class" : "bus",
+      "claimed" : true,
+      "handle" : "DMI:0002",
+      "description" : "Motherboard",
+      "product" : "ABCD",
+      "vendor" : "ABCD",
+      "physid" : "0",
+      "version" : "1234",
+      "serial" : "1234",
+      "slot" : "NULL",
+      "children" : [
+        {
+          "id" : "memory:0",
+          "class" : "memory",
+          "claimed" : true,
+          "handle" : "DMI:004A",
+          "description" : "System Memory",
+          "physid" : "4a",
+          "slot" : "System board or motherboard",
+          "children" : [
+            {
+              "id" : "bank:0",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:004C",
+              "description" : "DIMM Synchronous 2133 MHz (0.5 ns)",
+              "product" : "36ASF2G72PZ-2G1A2",
+              "vendor" : "Micron",
+              "physid" : "0",
+              "serial" : "101B6543",
+              "slot" : "DIMM_A0",
+              "units" : "bytes",
+              "size" : 17179869184,
+              "width" : 64,
+              "clock" : 2133000000
+            },
+            {
+              "id" : "bank:1",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:004E",
+              "description" : "DIMM Synchronous [empty]",
+              "product" : "NO DIMM",
+              "vendor" : "NO DIMM",
+              "physid" : "1",
+              "serial" : "NO DIMM",
+              "slot" : "DIMM_A1"
+            },
+            {
+              "id" : "bank:2",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:004F",
+              "description" : "DIMM Synchronous 2133 MHz (0.5 ns)",
+              "product" : "36ASF2G72PZ-2G1A2",
+              "vendor" : "Micron",
+              "physid" : "2",
+              "serial" : "101B654E",
+              "slot" : "DIMM_A2",
+              "units" : "bytes",
+              "size" : 17179869184,
+              "width" : 64,
+              "clock" : 2133000000
+            },
+            {
+              "id" : "bank:3",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:0051",
+              "description" : "DIMM Synchronous [empty]",
+              "product" : "NO DIMM",
+              "vendor" : "NO DIMM",
+              "physid" : "3",
+              "serial" : "NO DIMM",
+              "slot" : "DIMM_A3"
+            }
+          ]
+        },
+        {
+          "id" : "memory:1",
+          "class" : "memory",
+          "claimed" : true,
+          "handle" : "DMI:0052",
+          "description" : "System Memory",
+          "physid" : "52",
+          "slot" : "System board or motherboard",
+          "children" : [
+            {
+              "id" : "bank:0",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:0054",
+              "description" : "DIMM Synchronous 2133 MHz (0.5 ns)",
+              "product" : "36ASF2G72PZ-2G1A2",
+              "vendor" : "Micron",
+              "physid" : "0",
+              "serial" : "101B6545",
+              "slot" : "DIMM_A4",
+              "units" : "bytes",
+              "size" : 17179869184,
+              "width" : 64,
+              "clock" : 2133000000
+            },
+            {
+              "id" : "bank:1",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:0056",
+              "description" : "DIMM Synchronous [empty]",
+              "product" : "NO DIMM",
+              "vendor" : "NO DIMM",
+              "physid" : "1",
+              "serial" : "NO DIMM",
+              "slot" : "DIMM_A5"
+            },
+            {
+              "id" : "bank:2",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:0057",
+              "description" : "DIMM Synchronous 2133 MHz (0.5 ns)",
+              "product" : "36ASF2G72PZ-2G1A2",
+              "vendor" : "Micron",
+              "physid" : "2",
+              "serial" : "101B6540",
+              "slot" : "DIMM_A6",
+              "units" : "bytes",
+              "size" : 17179869184,
+              "width" : 64,
+              "clock" : 2133000000
+            },
+            {
+              "id" : "bank:3",
+              "class" : "memory",
+              "claimed" : true,
+              "handle" : "DMI:0059",
+              "description" : "DIMM Synchronous [empty]",
+              "product" : "NO DIMM",
+              "vendor" : "NO DIMM",
+              "physid" : "3",
+              "serial" : "NO DIMM",
+              "slot" : "DIMM_A7"
+            }
+          ]
+        },
+        {
+          "id" : "memory:4",
+          "class" : "memory",
+          "physid" : "1"
+        },
+        {
+          "id" : "memory:5",
+          "class" : "memory",
+          "physid" : "2"
+        }
+      ]
+    }
+  ]
+}
+""", "")
+
 SMARTCTL_NORMAL_OUTPUT = ("""
 smartctl 6.2 2017-02-27 r4394 [x86_64-linux-3.10.0-693.21.1.el7.x86_64] (local build)
 Copyright (C) 2002-13, Bruce Allen, Christian Franke, www.smartmontools.org
@@ -1070,9 +1258,9 @@ class TestGenericHardwareManager(base.IronicAgentTest):
 
     @mock.patch('psutil.virtual_memory', autospec=True)
     @mock.patch.object(utils, 'execute', autospec=True)
-    def test_get_memory_psutil(self, mocked_execute, mocked_psutil):
+    def test_get_memory_psutil_v1(self, mocked_execute, mocked_psutil):
         mocked_psutil.return_value.total = 3952 * 1024 * 1024
-        mocked_execute.return_value = LSHW_JSON_OUTPUT
+        mocked_execute.return_value = LSHW_JSON_OUTPUT_V1
         mem = self.hardware.get_memory()
 
         self.assertEqual(3952 * 1024 * 1024, mem.total)
@@ -1080,14 +1268,36 @@ class TestGenericHardwareManager(base.IronicAgentTest):
 
     @mock.patch('psutil.virtual_memory', autospec=True)
     @mock.patch.object(utils, 'execute', autospec=True)
-    def test_get_memory_psutil_exception(self, mocked_execute, mocked_psutil):
+    def test_get_memory_psutil_v2(self, mocked_execute, mocked_psutil):
-        mocked_execute.return_value = LSHW_JSON_OUTPUT
+        mocked_psutil.return_value.total = 3952 * 1024 * 1024
+        mocked_execute.return_value = LSHW_JSON_OUTPUT_V2
+        mem = self.hardware.get_memory()
+
+        self.assertEqual(3952 * 1024 * 1024, mem.total)
+        self.assertEqual(65536, mem.physical_mb)
+
+    @mock.patch('psutil.virtual_memory', autospec=True)
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_memory_psutil_exception_v1(self, mocked_execute,
+                                            mocked_psutil):
+        mocked_execute.return_value = LSHW_JSON_OUTPUT_V1
         mocked_psutil.side_effect = AttributeError()
         mem = self.hardware.get_memory()
 
         self.assertIsNone(mem.total)
         self.assertEqual(4096, mem.physical_mb)
 
+    @mock.patch('psutil.virtual_memory', autospec=True)
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_memory_psutil_exception_v2(self, mocked_execute,
+                                            mocked_psutil):
+        mocked_execute.return_value = LSHW_JSON_OUTPUT_V2
+        mocked_psutil.side_effect = AttributeError()
+        mem = self.hardware.get_memory()
+
+        self.assertIsNone(mem.total)
+        self.assertEqual(65536, mem.physical_mb)
+
     @mock.patch('psutil.virtual_memory', autospec=True)
     @mock.patch.object(utils, 'execute', autospec=True)
     def test_get_memory_lshw_exception(self, mocked_execute, mocked_psutil):
@@ -2087,7 +2297,7 @@ class TestGenericHardwareManager(base.IronicAgentTest):
 
     @mock.patch.object(utils, 'execute', autospec=True)
     def test_get_system_vendor_info(self, mocked_execute):
-        mocked_execute.return_value = LSHW_JSON_OUTPUT
+        mocked_execute.return_value = LSHW_JSON_OUTPUT_V1
         vendor_info = self.hardware.get_system_vendor_info()
         self.assertEqual('ABC123 (GENERIC_SERVER)', vendor_info.product_name)
         self.assertEqual('1234567', vendor_info.serial_number)

releasenotes/notes/get-physical-memory-535a32362bcdf83a.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes an issue with retrieving all available physical memory.
+    For more details see:
+    `story 2005308 <https://storyboard.openstack.org/#!/story/2005308>`