From 9e4c7052a2fd9aac03858db696bf1ea9487f15e6 Mon Sep 17 00:00:00 2001
From: Julia Kreger <juliaashleykreger@gmail.com>
Date: Wed, 26 May 2021 11:55:44 -0700
Subject: [PATCH] Limit qemu-img execution arenas

qemu-img attempts to launch multiple threads by default *and*
attempts to have multiple memory allocation arenas to operate
from. While multithreading can be good for performance, this
pattern and the memory footprint for process launch and
dependencies can turn the memory footprint for a cirros image
conversion (16MB) into 1.2GB of memory being asked for by the
qemu-img tool.

In order to limit this impact, as the default number of arenas
is governed by the number of CPUs times the number 8, it seems
reasonable to lower this to a more reasonable number which
also helps keep our possible memory footprint from being exceeded.

Change-Id: I71a28ec59ec31c691205eb34d9fcab63a2ccb682
Story: 2008928
Task: 42528
---
 ironic_python_agent/extensions/standby.py            | 12 +++++++++++-
 .../tests/unit/extensions/test_standby.py            |  4 +++-
 ...limit-qemu-img-malloc-arena-025ed84115481eae.yaml |  7 +++++++
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/limit-qemu-img-malloc-arena-025ed84115481eae.yaml

diff --git a/ironic_python_agent/extensions/standby.py b/ironic_python_agent/extensions/standby.py
index 74f244e0a..bdf338a9d 100644
--- a/ironic_python_agent/extensions/standby.py
+++ b/ironic_python_agent/extensions/standby.py
@@ -206,7 +206,17 @@ def _write_whole_disk_image(image, image_info, device):
         # TODO(dtantsur): switch to disk_utils.convert_image when it supports
         # -t and -W flags and defaults to 2 GiB memory limit.
         limits = processutils.ProcessLimits(address_space=2048 * units.Mi)
-        utils.execute(*command, prlimit=limits)
+        # TODO(TheJulia): qemu-img uses a default of 8 * nCPU to determine
+        # how many chunks of memory to allocate based upon the discussion in
+        # https://bugzilla.redhat.com/show_bug.cgi?id=1892773.
+        # Setting MALLOC_AREA_MAX=1 results in a process memory footprint of
+        # ~250mb. Setting it to 3 will allow for multiple areas, but won't
+        # greatly exceed the footprint.
+        env_vars = {'MALLOC_ARENA_MAX': '3'}
+        # Entirely disabling threading results in the same memory footprint
+        # as 1 areana, but multiple threads are useful for performance
+        # as the file could be fragmented/compressed.
+        utils.execute(*command, prlimit=limits, env_variables=env_vars)
     except processutils.ProcessExecutionError as e:
         raise errors.ImageWriteError(device, e.exit_code, e.stdout, e.stderr)
 
diff --git a/ironic_python_agent/tests/unit/extensions/test_standby.py b/ironic_python_agent/tests/unit/extensions/test_standby.py
index 6f905a36a..e75000275 100644
--- a/ironic_python_agent/tests/unit/extensions/test_standby.py
+++ b/ironic_python_agent/tests/unit/extensions/test_standby.py
@@ -182,7 +182,9 @@ class TestStandbyExtension(base.IronicAgentTest):
 
         standby._write_image(image_info, device)
 
-        execute_mock.assert_called_once_with(*command, prlimit=mock.ANY)
+        execute_mock.assert_called_once_with(
+            *command, prlimit=mock.ANY,
+            env_variables={'MALLOC_ARENA_MAX': '3'})
         wipe_mock.assert_called_once_with(device, '')
         udev_mock.assert_called_once_with()
         rescan_mock.assert_called_once_with(device)
diff --git a/releasenotes/notes/limit-qemu-img-malloc-arena-025ed84115481eae.yaml b/releasenotes/notes/limit-qemu-img-malloc-arena-025ed84115481eae.yaml
new file mode 100644
index 000000000..38ddf24ea
--- /dev/null
+++ b/releasenotes/notes/limit-qemu-img-malloc-arena-025ed84115481eae.yaml
@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixes failures with disk image conversions which result in memory
+    allocation or input/output errors due to memory limitations by limiting
+    the number of available memory allocation pools to a non-dynamic
+    reasonable number which should not exceed the available system memory.