Retry "lvs" call on segfault for _get_thin_pool_free_space
This is a follow-up to I6824ba4f. LVM commands segfault occasionally, exiting with code 139. Change I6824ba4f introduced a workaround to retry the command when code 139 is returned, which generally works. This expands that retry to the case where thin pool space is queried, which currently results in the LVM driver reporting no free space to the scheduler. Further work is needed to expand this to other LVM calls, but this patch is narrow in scope to target a particular gate failure. Related-Bug: #1901783 Partial-Bug: #1932188 Closes-Bug: #1932287 Change-Id: I0a2420f3e4a411f5fa52ebe2d22859b138ef387f
This commit is contained in:
parent
0caaa1fee5
commit
2da7d42b91
|
@ -157,6 +157,23 @@ class LVM(executor.Executor):
|
|||
def _create_vg(self, pv_list):
|
||||
cinder.privsep.lvm.create_vg(self.vg_name, pv_list)
|
||||
|
||||
@utils.retry(retry=utils.retry_if_exit_code, retry_param=139, interval=0.5,
|
||||
backoff_rate=0.5)
|
||||
def _run_lvm_command(self,
|
||||
cmd_arg_list: list,
|
||||
root_helper: str,
|
||||
run_as_root: bool = True) -> tuple:
|
||||
"""Run LVM commands with a retry on code 139 to work around LVM bugs.
|
||||
|
||||
Refer to LP bug 1901783, LP bug 1932188.
|
||||
"""
|
||||
|
||||
(out, err) = self._execute(*cmd_arg_list,
|
||||
root_helper=root_helper,
|
||||
run_as_root=run_as_root)
|
||||
|
||||
return (out, err)
|
||||
|
||||
def _get_thin_pool_free_space(self, vg_name, thin_pool_name):
|
||||
"""Returns available thin pool free space.
|
||||
|
||||
|
@ -176,9 +193,9 @@ class LVM(executor.Executor):
|
|||
free_space = 0.0
|
||||
|
||||
try:
|
||||
(out, err) = self._execute(*cmd,
|
||||
root_helper=self._root_helper,
|
||||
run_as_root=True)
|
||||
(out, err) = self._run_lvm_command(cmd,
|
||||
root_helper=self._root_helper,
|
||||
run_as_root=True)
|
||||
if out is not None:
|
||||
out = out.strip()
|
||||
data = out.split(':')
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
from unittest import mock
|
||||
|
||||
import ddt
|
||||
from os_brick import executor as os_brick_executor
|
||||
from oslo_concurrency import processutils
|
||||
|
||||
from cinder.brick.local_dev import lvm as brick
|
||||
|
@ -252,6 +253,28 @@ class BrickLvmTestCase(test.TestCase):
|
|||
lvs_call = mock.call(*args, root_helper='sudo', run_as_root=True)
|
||||
exec_mock.assert_has_calls([lvs_call, lvs_call])
|
||||
|
||||
@mock.patch('tenacity.nap.sleep', mock.Mock())
|
||||
@mock.patch.object(os_brick_executor.Executor, '_execute')
|
||||
def test_get_thin_pool_free_space_retry(self, exec_mock):
|
||||
exec_mock.side_effect = (
|
||||
processutils.ProcessExecutionError('', '', exit_code=139),
|
||||
('15.84:50', ''),
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
7.92,
|
||||
self.vg._get_thin_pool_free_space('vg', 'thinpool')
|
||||
)
|
||||
|
||||
self.assertEqual(2, exec_mock.call_count)
|
||||
args = ['env', 'LC_ALL=C', 'lvs', '--noheadings', '--unit=g', '-o',
|
||||
'size,data_percent', '--separator', ':', '--nosuffix',
|
||||
'/dev/vg/thinpool']
|
||||
if self.configuration.lvm_suppress_fd_warnings:
|
||||
args.insert(2, 'LVM_SUPPRESS_FD_WARNINGS=1')
|
||||
lvs_call = mock.call(*args, root_helper='sudo', run_as_root=True)
|
||||
exec_mock.assert_has_calls([lvs_call, lvs_call])
|
||||
|
||||
def test_get_all_physical_volumes(self):
|
||||
# Filtered VG version
|
||||
pvs = self.vg.get_all_physical_volumes('sudo', 'fake-vg')
|
||||
|
|
Loading…
Reference in New Issue