Merge "Partially revert "Pull libc-related functions out to a separate module""
This commit is contained in:
Zuul
Gerrit Code Review
commit
a8e77ac19f
@ -39,6 +39,8 @@ import functools
|
||||
import email.parser
|
||||
from random import random, shuffle
|
||||
from contextlib import contextmanager, closing
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
from optparse import OptionParser
|
||||
import traceback
|
||||
import warnings
|
||||
@ -94,13 +96,10 @@ from swift.common.registry import register_swift_info, get_swift_info # noqa
|
||||
from swift.common.utils.libc import ( # noqa
|
||||
F_SETPIPE_SZ,
|
||||
load_libc_function,
|
||||
config_fallocate_value,
|
||||
disable_fallocate,
|
||||
fallocate,
|
||||
punch_hole,
|
||||
drop_buffer_cache,
|
||||
get_md5_socket,
|
||||
modify_priority,
|
||||
_LibcWrapper,
|
||||
)
|
||||
from swift.common.utils.timestamp import ( # noqa
|
||||
NORMAL_FORMAT,
|
||||
@ -129,6 +128,21 @@ import logging
|
||||
|
||||
NOTICE = 25
|
||||
|
||||
# These are lazily pulled from libc elsewhere
|
||||
_sys_fallocate = None
|
||||
|
||||
# If set to non-zero, fallocate routines will fail based on free space
|
||||
# available being at or below this amount, in bytes.
|
||||
FALLOCATE_RESERVE = 0
|
||||
# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or
|
||||
# the number of bytes (False).
|
||||
FALLOCATE_IS_PERCENT = False
|
||||
|
||||
# from /usr/include/linux/falloc.h
|
||||
FALLOC_FL_KEEP_SIZE = 1
|
||||
FALLOC_FL_PUNCH_HOLE = 2
|
||||
|
||||
|
||||
# Used by hash_path to offer a bit more security when generating hashes for
|
||||
# paths. It simply appends this value to all paths; guessing the hash a path
|
||||
# will end up with would also require knowing this suffix.
|
||||
@ -137,6 +151,10 @@ HASH_PATH_PREFIX = b''
|
||||
|
||||
SWIFT_CONF_FILE = '/etc/swift/swift.conf'
|
||||
|
||||
# These constants are Linux-specific, and Python doesn't seem to know
|
||||
# about them. We ask anyway just in case that ever gets fixed.
|
||||
#
|
||||
# The values were copied from the Linux 3.x kernel headers.
|
||||
O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY)
|
||||
|
||||
MD5_OF_EMPTY_STRING = 'd41d8cd98f00b204e9800998ecf8427e'
|
||||
@ -672,6 +690,25 @@ def get_trans_id_time(trans_id):
|
||||
return None
|
||||
|
||||
|
||||
def config_fallocate_value(reserve_value):
|
||||
"""
|
||||
Returns fallocate reserve_value as an int or float.
|
||||
Returns is_percent as a boolean.
|
||||
Returns a ValueError on invalid fallocate value.
|
||||
"""
|
||||
try:
|
||||
if str(reserve_value[-1:]) == '%':
|
||||
reserve_value = float(reserve_value[:-1])
|
||||
is_percent = True
|
||||
else:
|
||||
reserve_value = int(reserve_value)
|
||||
is_percent = False
|
||||
except ValueError:
|
||||
raise ValueError('Error: %s is an invalid value for fallocate'
|
||||
'_reserve.' % reserve_value)
|
||||
return reserve_value, is_percent
|
||||
|
||||
|
||||
class FileLikeIter(object):
|
||||
|
||||
def __init__(self, iterable):
|
||||
@ -822,6 +859,116 @@ def fs_has_free_space(fs_path, space_needed, is_percent):
|
||||
return free_bytes >= space_needed
|
||||
|
||||
|
||||
_fallocate_enabled = True
|
||||
_fallocate_warned_about_missing = False
|
||||
_sys_fallocate = _LibcWrapper('fallocate')
|
||||
_sys_posix_fallocate = _LibcWrapper('posix_fallocate')
|
||||
|
||||
|
||||
def disable_fallocate():
|
||||
global _fallocate_enabled
|
||||
_fallocate_enabled = False
|
||||
|
||||
|
||||
def fallocate(fd, size, offset=0):
|
||||
"""
|
||||
Pre-allocate disk space for a file.
|
||||
|
||||
This function can be disabled by calling disable_fallocate(). If no
|
||||
suitable C function is available in libc, this function is a no-op.
|
||||
|
||||
:param fd: file descriptor
|
||||
:param size: size to allocate (in bytes)
|
||||
"""
|
||||
global _fallocate_enabled
|
||||
if not _fallocate_enabled:
|
||||
return
|
||||
|
||||
if size < 0:
|
||||
size = 0 # Done historically; not really sure why
|
||||
if size >= (1 << 63):
|
||||
raise ValueError('size must be less than 2 ** 63')
|
||||
if offset < 0:
|
||||
raise ValueError('offset must be non-negative')
|
||||
if offset >= (1 << 63):
|
||||
raise ValueError('offset must be less than 2 ** 63')
|
||||
|
||||
# Make sure there's some (configurable) amount of free space in
|
||||
# addition to the number of bytes we're allocating.
|
||||
if FALLOCATE_RESERVE:
|
||||
st = os.fstatvfs(fd)
|
||||
free = st.f_frsize * st.f_bavail - size
|
||||
if FALLOCATE_IS_PERCENT:
|
||||
free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
|
||||
if float(free) <= float(FALLOCATE_RESERVE):
|
||||
raise OSError(
|
||||
errno.ENOSPC,
|
||||
'FALLOCATE_RESERVE fail %g <= %g' %
|
||||
(free, FALLOCATE_RESERVE))
|
||||
|
||||
if _sys_fallocate.available:
|
||||
# Parameters are (fd, mode, offset, length).
|
||||
#
|
||||
# mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without
|
||||
# affecting the reported file size).
|
||||
ret = _sys_fallocate(
|
||||
fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset),
|
||||
ctypes.c_uint64(size))
|
||||
err = ctypes.get_errno()
|
||||
elif _sys_posix_fallocate.available:
|
||||
# Parameters are (fd, offset, length).
|
||||
ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset),
|
||||
ctypes.c_uint64(size))
|
||||
err = ctypes.get_errno()
|
||||
else:
|
||||
# No suitable fallocate-like function is in our libc. Warn about it,
|
||||
# but just once per process, and then do nothing.
|
||||
global _fallocate_warned_about_missing
|
||||
if not _fallocate_warned_about_missing:
|
||||
logging.warning("Unable to locate fallocate, posix_fallocate in "
|
||||
"libc. Leaving as a no-op.")
|
||||
_fallocate_warned_about_missing = True
|
||||
return
|
||||
|
||||
if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP,
|
||||
errno.EINVAL):
|
||||
raise OSError(err, 'Unable to fallocate(%s)' % size)
|
||||
|
||||
|
||||
def punch_hole(fd, offset, length):
|
||||
"""
|
||||
De-allocate disk space in the middle of a file.
|
||||
|
||||
:param fd: file descriptor
|
||||
:param offset: index of first byte to de-allocate
|
||||
:param length: number of bytes to de-allocate
|
||||
"""
|
||||
if offset < 0:
|
||||
raise ValueError('offset must be non-negative')
|
||||
if offset >= (1 << 63):
|
||||
raise ValueError('offset must be less than 2 ** 63')
|
||||
if length <= 0:
|
||||
raise ValueError('length must be positive')
|
||||
if length >= (1 << 63):
|
||||
raise ValueError('length must be less than 2 ** 63')
|
||||
|
||||
if _sys_fallocate.available:
|
||||
# Parameters are (fd, mode, offset, length).
|
||||
ret = _sys_fallocate(
|
||||
fd,
|
||||
FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
|
||||
ctypes.c_uint64(offset),
|
||||
ctypes.c_uint64(length))
|
||||
err = ctypes.get_errno()
|
||||
if ret and err:
|
||||
mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE"
|
||||
raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % (
|
||||
fd, mode_str, offset, length))
|
||||
else:
|
||||
raise OSError(errno.ENOTSUP,
|
||||
'No suitable C function found for hole punching')
|
||||
|
||||
|
||||
def fsync(fd):
|
||||
"""
|
||||
Sync modified file data and metadata to disk.
|
||||
|
||||
@ -17,7 +17,6 @@
|
||||
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import errno
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
@ -26,7 +25,6 @@ import socket
|
||||
|
||||
|
||||
# These are lazily pulled from libc elsewhere
|
||||
_sys_fallocate = None
|
||||
_posix_fadvise = None
|
||||
_libc_socket = None
|
||||
_libc_bind = None
|
||||
@ -36,17 +34,6 @@ _libc_setpriority = None
|
||||
# see man -s 2 syscall
|
||||
_posix_syscall = None
|
||||
|
||||
# If set to non-zero, fallocate routines will fail based on free space
|
||||
# available being at or below this amount, in bytes.
|
||||
FALLOCATE_RESERVE = 0
|
||||
# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or
|
||||
# the number of bytes (False).
|
||||
FALLOCATE_IS_PERCENT = False
|
||||
|
||||
# from /usr/include/linux/falloc.h
|
||||
FALLOC_FL_KEEP_SIZE = 1
|
||||
FALLOC_FL_PUNCH_HOLE = 2
|
||||
|
||||
# from /usr/src/linux-headers-*/include/uapi/linux/resource.h
|
||||
PRIO_PROCESS = 0
|
||||
|
||||
@ -191,135 +178,6 @@ class _LibcWrapper(object):
|
||||
"No function %r found in libc" % self._func_name)
|
||||
|
||||
|
||||
def config_fallocate_value(reserve_value):
|
||||
"""
|
||||
Returns fallocate reserve_value as an int or float.
|
||||
Returns is_percent as a boolean.
|
||||
Returns a ValueError on invalid fallocate value.
|
||||
"""
|
||||
try:
|
||||
if str(reserve_value[-1:]) == '%':
|
||||
reserve_value = float(reserve_value[:-1])
|
||||
is_percent = True
|
||||
else:
|
||||
reserve_value = int(reserve_value)
|
||||
is_percent = False
|
||||
except ValueError:
|
||||
raise ValueError('Error: %s is an invalid value for fallocate'
|
||||
'_reserve.' % reserve_value)
|
||||
return reserve_value, is_percent
|
||||
|
||||
|
||||
_fallocate_enabled = True
|
||||
_fallocate_warned_about_missing = False
|
||||
_sys_fallocate = _LibcWrapper('fallocate')
|
||||
_sys_posix_fallocate = _LibcWrapper('posix_fallocate')
|
||||
|
||||
|
||||
def disable_fallocate():
|
||||
global _fallocate_enabled
|
||||
_fallocate_enabled = False
|
||||
|
||||
|
||||
def fallocate(fd, size, offset=0):
|
||||
"""
|
||||
Pre-allocate disk space for a file.
|
||||
|
||||
This function can be disabled by calling disable_fallocate(). If no
|
||||
suitable C function is available in libc, this function is a no-op.
|
||||
|
||||
:param fd: file descriptor
|
||||
:param size: size to allocate (in bytes)
|
||||
"""
|
||||
global _fallocate_enabled
|
||||
if not _fallocate_enabled:
|
||||
return
|
||||
|
||||
if size < 0:
|
||||
size = 0 # Done historically; not really sure why
|
||||
if size >= (1 << 63):
|
||||
raise ValueError('size must be less than 2 ** 63')
|
||||
if offset < 0:
|
||||
raise ValueError('offset must be non-negative')
|
||||
if offset >= (1 << 63):
|
||||
raise ValueError('offset must be less than 2 ** 63')
|
||||
|
||||
# Make sure there's some (configurable) amount of free space in
|
||||
# addition to the number of bytes we're allocating.
|
||||
if FALLOCATE_RESERVE:
|
||||
st = os.fstatvfs(fd)
|
||||
free = st.f_frsize * st.f_bavail - size
|
||||
if FALLOCATE_IS_PERCENT:
|
||||
free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
|
||||
if float(free) <= float(FALLOCATE_RESERVE):
|
||||
raise OSError(
|
||||
errno.ENOSPC,
|
||||
'FALLOCATE_RESERVE fail %g <= %g' %
|
||||
(free, FALLOCATE_RESERVE))
|
||||
|
||||
if _sys_fallocate.available:
|
||||
# Parameters are (fd, mode, offset, length).
|
||||
#
|
||||
# mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without
|
||||
# affecting the reported file size).
|
||||
ret = _sys_fallocate(
|
||||
fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset),
|
||||
ctypes.c_uint64(size))
|
||||
err = ctypes.get_errno()
|
||||
elif _sys_posix_fallocate.available:
|
||||
# Parameters are (fd, offset, length).
|
||||
ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset),
|
||||
ctypes.c_uint64(size))
|
||||
err = ctypes.get_errno()
|
||||
else:
|
||||
# No suitable fallocate-like function is in our libc. Warn about it,
|
||||
# but just once per process, and then do nothing.
|
||||
global _fallocate_warned_about_missing
|
||||
if not _fallocate_warned_about_missing:
|
||||
logging.warning("Unable to locate fallocate, posix_fallocate in "
|
||||
"libc. Leaving as a no-op.")
|
||||
_fallocate_warned_about_missing = True
|
||||
return
|
||||
|
||||
if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP,
|
||||
errno.EINVAL):
|
||||
raise OSError(err, 'Unable to fallocate(%s)' % size)
|
||||
|
||||
|
||||
def punch_hole(fd, offset, length):
|
||||
"""
|
||||
De-allocate disk space in the middle of a file.
|
||||
|
||||
:param fd: file descriptor
|
||||
:param offset: index of first byte to de-allocate
|
||||
:param length: number of bytes to de-allocate
|
||||
"""
|
||||
if offset < 0:
|
||||
raise ValueError('offset must be non-negative')
|
||||
if offset >= (1 << 63):
|
||||
raise ValueError('offset must be less than 2 ** 63')
|
||||
if length <= 0:
|
||||
raise ValueError('length must be positive')
|
||||
if length >= (1 << 63):
|
||||
raise ValueError('length must be less than 2 ** 63')
|
||||
|
||||
if _sys_fallocate.available:
|
||||
# Parameters are (fd, mode, offset, length).
|
||||
ret = _sys_fallocate(
|
||||
fd,
|
||||
FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
|
||||
ctypes.c_uint64(offset),
|
||||
ctypes.c_uint64(length))
|
||||
err = ctypes.get_errno()
|
||||
if ret and err:
|
||||
mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE"
|
||||
raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % (
|
||||
fd, mode_str, offset, length))
|
||||
else:
|
||||
raise OSError(errno.ENOTSUP,
|
||||
'No suitable C function found for hole punching')
|
||||
|
||||
|
||||
def drop_buffer_cache(fd, offset, length):
|
||||
"""
|
||||
Drop 'buffer' cache for the given range of the given file.
|
||||
|
||||
@ -2712,6 +2712,44 @@ cluster_dfw1 = http://dfw1.host/v1/
|
||||
ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright')
|
||||
self.assertIsNone(ts)
|
||||
|
||||
def test_config_fallocate_value(self):
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('10%')
|
||||
self.assertEqual(fallocate_value, 10)
|
||||
self.assertTrue(is_percent)
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('10')
|
||||
self.assertEqual(fallocate_value, 10)
|
||||
self.assertFalse(is_percent)
|
||||
try:
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('ab%')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: ab% is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
try:
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('ab')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: ab is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
try:
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('1%%')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: 1%% is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
try:
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('10.0')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('10.5%')
|
||||
self.assertEqual(fallocate_value, 10.5)
|
||||
self.assertTrue(is_percent)
|
||||
fallocate_value, is_percent = utils.config_fallocate_value('10.000%')
|
||||
self.assertEqual(fallocate_value, 10.000)
|
||||
self.assertTrue(is_percent)
|
||||
|
||||
def test_lock_file(self):
|
||||
flags = os.O_CREAT | os.O_RDWR
|
||||
with NamedTemporaryFile(delete=False) as nt:
|
||||
@ -8785,6 +8823,397 @@ class TestShardRangeList(unittest.TestCase):
|
||||
do_test([utils.ShardRange.ACTIVE]))
|
||||
|
||||
|
||||
@patch('ctypes.get_errno')
|
||||
@patch.object(utils, '_sys_posix_fallocate')
|
||||
@patch.object(utils, '_sys_fallocate')
|
||||
@patch.object(utils, 'FALLOCATE_RESERVE', 0)
|
||||
class TestFallocate(unittest.TestCase):
|
||||
def test_fallocate(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
utils.fallocate(1234, 5000 * 2 ** 20)
|
||||
|
||||
# We can't use sys_fallocate_mock.assert_called_once_with because no
|
||||
# two ctypes.c_uint64 objects are equal even if their values are
|
||||
# equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123).
|
||||
calls = sys_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 4)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE)
|
||||
self.assertEqual(args[2].value, 0)
|
||||
self.assertEqual(args[3].value, 5000 * 2 ** 20)
|
||||
|
||||
sys_posix_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_fallocate_offset(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
|
||||
calls = sys_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 4)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE)
|
||||
self.assertEqual(args[2].value, 3 * 2 ** 30)
|
||||
self.assertEqual(args[3].value, 5000 * 2 ** 20)
|
||||
|
||||
sys_posix_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_fallocate_fatal_error(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = -1
|
||||
get_errno_mock.return_value = errno.EIO
|
||||
|
||||
with self.assertRaises(OSError) as cm:
|
||||
utils.fallocate(1234, 5000 * 2 ** 20)
|
||||
self.assertEqual(cm.exception.errno, errno.EIO)
|
||||
|
||||
def test_fallocate_silent_errors(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = -1
|
||||
|
||||
for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL):
|
||||
get_errno_mock.return_value = silent_error
|
||||
try:
|
||||
utils.fallocate(1234, 5678)
|
||||
except OSError:
|
||||
self.fail("fallocate() raised an error on %d", silent_error)
|
||||
|
||||
def test_posix_fallocate_fallback(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock,
|
||||
get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
sys_fallocate_mock.side_effect = NotImplementedError
|
||||
|
||||
sys_posix_fallocate_mock.available = True
|
||||
sys_posix_fallocate_mock.return_value = 0
|
||||
|
||||
utils.fallocate(1234, 567890)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
calls = sys_posix_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 3)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1].value, 0)
|
||||
self.assertEqual(args[2].value, 567890)
|
||||
|
||||
def test_posix_fallocate_offset(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
sys_fallocate_mock.side_effect = NotImplementedError
|
||||
|
||||
sys_posix_fallocate_mock.available = True
|
||||
sys_posix_fallocate_mock.return_value = 0
|
||||
|
||||
utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
|
||||
calls = sys_posix_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 3)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1].value, 3 * 2 ** 30)
|
||||
self.assertEqual(args[2].value, 5000 * 2 ** 20)
|
||||
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_no_fallocates_available(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
sys_posix_fallocate_mock.available = False
|
||||
|
||||
with mock.patch("logging.warning") as warning_mock, \
|
||||
mock.patch.object(utils, "_fallocate_warned_about_missing",
|
||||
False):
|
||||
utils.fallocate(321, 654)
|
||||
utils.fallocate(321, 654)
|
||||
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
sys_posix_fallocate_mock.assert_not_called()
|
||||
get_errno_mock.assert_not_called()
|
||||
|
||||
self.assertEqual(len(warning_mock.mock_calls), 1)
|
||||
|
||||
def test_arg_bounds(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
with self.assertRaises(ValueError):
|
||||
utils.fallocate(0, 1 << 64, 0)
|
||||
with self.assertRaises(ValueError):
|
||||
utils.fallocate(0, 0, -1)
|
||||
with self.assertRaises(ValueError):
|
||||
utils.fallocate(0, 0, 1 << 64)
|
||||
self.assertEqual([], sys_fallocate_mock.mock_calls)
|
||||
# sanity check
|
||||
utils.fallocate(0, 0, 0)
|
||||
self.assertEqual(
|
||||
[mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
|
||||
sys_fallocate_mock.mock_calls)
|
||||
# Go confirm the ctypes values separately; apparently == doesn't
|
||||
# work the way you'd expect with ctypes :-/
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
|
||||
sys_fallocate_mock.reset_mock()
|
||||
|
||||
# negative size will be adjusted as 0
|
||||
utils.fallocate(0, -1, 0)
|
||||
self.assertEqual(
|
||||
[mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
|
||||
sys_fallocate_mock.mock_calls)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
|
||||
|
||||
|
||||
@patch.object(os, 'fstatvfs')
|
||||
@patch.object(utils, '_sys_fallocate', available=True, return_value=0)
|
||||
@patch.object(utils, 'FALLOCATE_RESERVE', 0)
|
||||
@patch.object(utils, 'FALLOCATE_IS_PERCENT', False)
|
||||
@patch.object(utils, '_fallocate_enabled', True)
|
||||
class TestFallocateReserve(unittest.TestCase):
|
||||
def _statvfs_result(self, f_frsize, f_bavail):
|
||||
# Only 3 values are relevant to us, so use zeros for the rest
|
||||
f_blocks = 100
|
||||
return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail,
|
||||
0, 0, 0, 0, 0))
|
||||
|
||||
def test_disabled(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
utils.disable_fallocate()
|
||||
utils.fallocate(123, 456)
|
||||
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
fstatvfs_mock.assert_not_called()
|
||||
|
||||
def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
utils.fallocate(123, 456)
|
||||
|
||||
fstatvfs_mock.assert_not_called()
|
||||
self.assertEqual(len(sys_fallocate_mock.mock_calls), 1)
|
||||
|
||||
def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks
|
||||
# of size 1024 free, so succeed
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
|
||||
utils.fallocate(88, 1023)
|
||||
|
||||
def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks
|
||||
# of size 1024 free, so fail
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
utils.fallocate(88, 1024)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks
|
||||
# of size 1024 free, so fail
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
utils.fallocate(88, 1 << 30)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail %g <= 1024'
|
||||
% (errno.ENOSPC, ((2 * 1024) - (1 << 30))))
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_enough_space_small_blocks(self, sys_fallocate_mock,
|
||||
fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks
|
||||
# of size 512 free, so succeed
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
|
||||
utils.fallocate(88, 1023)
|
||||
|
||||
def test_not_enough_space_small_blocks(self, sys_fallocate_mock,
|
||||
fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks
|
||||
# of size 512 free, so fail
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
utils.fallocate(88, 1024)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 2048 bytes in reserve but have only 3 blocks of size 512, so
|
||||
# allocating even 0 bytes fails
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('2048')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(512, 3)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
utils.fallocate(88, 0)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Filesystem is empty, but our reserve is bigger than the
|
||||
# filesystem, so any allocation will fail
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('9999999999999')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
|
||||
self.assertRaises(OSError, utils.fallocate, 88, 0)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
|
||||
# and file size is 2047, so succeed
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1%')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
|
||||
utils.fallocate(88, 2047)
|
||||
|
||||
def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
|
||||
# and file size is 2048, so fail
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('1%')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
utils.fallocate(88, 2048)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1 <= 1'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Filesystem is empty, but our reserve is the whole filesystem, so
|
||||
# any allocation will fail
|
||||
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
|
||||
utils.config_fallocate_value('100%')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
utils.fallocate(88, 0)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 100 <= 100'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
|
||||
@patch('ctypes.get_errno')
|
||||
@patch.object(utils, '_sys_fallocate')
|
||||
class TestPunchHole(unittest.TestCase):
|
||||
def test_punch_hole(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
utils.punch_hole(123, 456, 789)
|
||||
|
||||
calls = sys_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 4)
|
||||
self.assertEqual(args[0], 123)
|
||||
self.assertEqual(
|
||||
args[1], utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE)
|
||||
self.assertEqual(args[2].value, 456)
|
||||
self.assertEqual(args[3].value, 789)
|
||||
|
||||
def test_error(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = -1
|
||||
get_errno_mock.return_value = errno.EISDIR
|
||||
|
||||
with self.assertRaises(OSError) as cm:
|
||||
utils.punch_hole(123, 456, 789)
|
||||
self.assertEqual(cm.exception.errno, errno.EISDIR)
|
||||
|
||||
def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
utils.punch_hole(0, 1, -1)
|
||||
with self.assertRaises(ValueError):
|
||||
utils.punch_hole(0, 1 << 64, 1)
|
||||
with self.assertRaises(ValueError):
|
||||
utils.punch_hole(0, -1, 1)
|
||||
with self.assertRaises(ValueError):
|
||||
utils.punch_hole(0, 1, 0)
|
||||
with self.assertRaises(ValueError):
|
||||
utils.punch_hole(0, 1, 1 << 64)
|
||||
self.assertEqual([], sys_fallocate_mock.mock_calls)
|
||||
|
||||
# sanity check
|
||||
utils.punch_hole(0, 0, 1)
|
||||
self.assertEqual(
|
||||
[mock.call(
|
||||
0, utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE,
|
||||
mock.ANY, mock.ANY)],
|
||||
sys_fallocate_mock.mock_calls)
|
||||
# Go confirm the ctypes values separately; apparently == doesn't
|
||||
# work the way you'd expect with ctypes :-/
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1)
|
||||
|
||||
def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
|
||||
with self.assertRaises(OSError) as cm:
|
||||
utils.punch_hole(123, 456, 789)
|
||||
self.assertEqual(cm.exception.errno, errno.ENOTSUP)
|
||||
|
||||
|
||||
class TestPunchHoleReally(unittest.TestCase):
|
||||
def setUp(self):
|
||||
if not utils._sys_fallocate.available:
|
||||
raise unittest.SkipTest("utils._sys_fallocate not available")
|
||||
|
||||
def test_punch_a_hole(self):
|
||||
with TemporaryFile() as tf:
|
||||
tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64)
|
||||
tf.flush()
|
||||
|
||||
# knock out the first half of the "y"s
|
||||
utils.punch_hole(tf.fileno(), 64, 32)
|
||||
|
||||
tf.seek(0)
|
||||
contents = tf.read(4096)
|
||||
self.assertEqual(
|
||||
contents,
|
||||
b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64)
|
||||
|
||||
|
||||
class TestWatchdog(unittest.TestCase):
|
||||
def test_start_stop(self):
|
||||
w = utils.Watchdog()
|
||||
|
||||
@ -16,10 +16,8 @@
|
||||
"""Tests for swift.common.utils.libc"""
|
||||
|
||||
import ctypes
|
||||
import errno
|
||||
import os
|
||||
import platform
|
||||
import posix
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
@ -30,436 +28,6 @@ from swift.common.utils import libc
|
||||
from test.debug_logger import debug_logger
|
||||
|
||||
|
||||
@mock.patch('ctypes.get_errno')
|
||||
@mock.patch.object(libc, '_sys_posix_fallocate')
|
||||
@mock.patch.object(libc, '_sys_fallocate')
|
||||
@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0)
|
||||
class TestFallocate(unittest.TestCase):
|
||||
def test_config_fallocate_value(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('10%')
|
||||
self.assertEqual(fallocate_value, 10)
|
||||
self.assertTrue(is_percent)
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('10')
|
||||
self.assertEqual(fallocate_value, 10)
|
||||
self.assertFalse(is_percent)
|
||||
try:
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('ab%')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: ab% is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
try:
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('ab')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: ab is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
try:
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('1%%')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: 1%% is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
try:
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('10.0')
|
||||
except ValueError as err:
|
||||
exc = err
|
||||
self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for '
|
||||
'fallocate_reserve.')
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('10.5%')
|
||||
self.assertEqual(fallocate_value, 10.5)
|
||||
self.assertTrue(is_percent)
|
||||
fallocate_value, is_percent = libc.config_fallocate_value('10.000%')
|
||||
self.assertEqual(fallocate_value, 10.000)
|
||||
self.assertTrue(is_percent)
|
||||
|
||||
def test_fallocate(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
libc.fallocate(1234, 5000 * 2 ** 20)
|
||||
|
||||
# We can't use sys_fallocate_mock.assert_called_once_with because no
|
||||
# two ctypes.c_uint64 objects are equal even if their values are
|
||||
# equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123).
|
||||
calls = sys_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 4)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE)
|
||||
self.assertEqual(args[2].value, 0)
|
||||
self.assertEqual(args[3].value, 5000 * 2 ** 20)
|
||||
|
||||
sys_posix_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_fallocate_offset(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
|
||||
calls = sys_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 4)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE)
|
||||
self.assertEqual(args[2].value, 3 * 2 ** 30)
|
||||
self.assertEqual(args[3].value, 5000 * 2 ** 20)
|
||||
|
||||
sys_posix_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_fallocate_fatal_error(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = -1
|
||||
get_errno_mock.return_value = errno.EIO
|
||||
|
||||
with self.assertRaises(OSError) as cm:
|
||||
libc.fallocate(1234, 5000 * 2 ** 20)
|
||||
self.assertEqual(cm.exception.errno, errno.EIO)
|
||||
|
||||
def test_fallocate_silent_errors(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = -1
|
||||
|
||||
for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL):
|
||||
get_errno_mock.return_value = silent_error
|
||||
try:
|
||||
libc.fallocate(1234, 5678)
|
||||
except OSError:
|
||||
self.fail("fallocate() raised an error on %d", silent_error)
|
||||
|
||||
def test_posix_fallocate_fallback(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock,
|
||||
get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
sys_fallocate_mock.side_effect = NotImplementedError
|
||||
|
||||
sys_posix_fallocate_mock.available = True
|
||||
sys_posix_fallocate_mock.return_value = 0
|
||||
|
||||
libc.fallocate(1234, 567890)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
calls = sys_posix_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 3)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1].value, 0)
|
||||
self.assertEqual(args[2].value, 567890)
|
||||
|
||||
def test_posix_fallocate_offset(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
sys_fallocate_mock.side_effect = NotImplementedError
|
||||
|
||||
sys_posix_fallocate_mock.available = True
|
||||
sys_posix_fallocate_mock.return_value = 0
|
||||
|
||||
libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
|
||||
calls = sys_posix_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 3)
|
||||
self.assertEqual(args[0], 1234)
|
||||
self.assertEqual(args[1].value, 3 * 2 ** 30)
|
||||
self.assertEqual(args[2].value, 5000 * 2 ** 20)
|
||||
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_no_fallocates_available(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
sys_posix_fallocate_mock.available = False
|
||||
|
||||
with mock.patch("logging.warning") as warning_mock, \
|
||||
mock.patch.object(libc, "_fallocate_warned_about_missing",
|
||||
False):
|
||||
libc.fallocate(321, 654)
|
||||
libc.fallocate(321, 654)
|
||||
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
sys_posix_fallocate_mock.assert_not_called()
|
||||
get_errno_mock.assert_not_called()
|
||||
|
||||
self.assertEqual(len(warning_mock.mock_calls), 1)
|
||||
|
||||
def test_arg_bounds(self, sys_fallocate_mock,
|
||||
sys_posix_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
with self.assertRaises(ValueError):
|
||||
libc.fallocate(0, 1 << 64, 0)
|
||||
with self.assertRaises(ValueError):
|
||||
libc.fallocate(0, 0, -1)
|
||||
with self.assertRaises(ValueError):
|
||||
libc.fallocate(0, 0, 1 << 64)
|
||||
self.assertEqual([], sys_fallocate_mock.mock_calls)
|
||||
# sanity check
|
||||
libc.fallocate(0, 0, 0)
|
||||
self.assertEqual(
|
||||
[mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
|
||||
sys_fallocate_mock.mock_calls)
|
||||
# Go confirm the ctypes values separately; apparently == doesn't
|
||||
# work the way you'd expect with ctypes :-/
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
|
||||
sys_fallocate_mock.reset_mock()
|
||||
|
||||
# negative size will be adjusted as 0
|
||||
libc.fallocate(0, -1, 0)
|
||||
self.assertEqual(
|
||||
[mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
|
||||
sys_fallocate_mock.mock_calls)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
|
||||
|
||||
|
||||
@mock.patch.object(os, 'fstatvfs')
|
||||
@mock.patch.object(libc, '_sys_fallocate', available=True, return_value=0)
|
||||
@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0)
|
||||
@mock.patch.object(libc, 'FALLOCATE_IS_PERCENT', False)
|
||||
@mock.patch.object(libc, '_fallocate_enabled', True)
|
||||
class TestFallocateReserve(unittest.TestCase):
|
||||
def _statvfs_result(self, f_frsize, f_bavail):
|
||||
# Only 3 values are relevant to us, so use zeros for the rest
|
||||
f_blocks = 100
|
||||
return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail,
|
||||
0, 0, 0, 0, 0))
|
||||
|
||||
def test_disabled(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
libc.disable_fallocate()
|
||||
libc.fallocate(123, 456)
|
||||
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
fstatvfs_mock.assert_not_called()
|
||||
|
||||
def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
libc.fallocate(123, 456)
|
||||
|
||||
fstatvfs_mock.assert_not_called()
|
||||
self.assertEqual(len(sys_fallocate_mock.mock_calls), 1)
|
||||
|
||||
def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks
|
||||
# of size 1024 free, so succeed
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
|
||||
libc.fallocate(88, 1023)
|
||||
|
||||
def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks
|
||||
# of size 1024 free, so fail
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
libc.fallocate(88, 1024)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks
|
||||
# of size 1024 free, so fail
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
libc.fallocate(88, 1 << 30)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail %g <= 1024'
|
||||
% (errno.ENOSPC, ((2 * 1024) - (1 << 30))))
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_enough_space_small_blocks(self, sys_fallocate_mock,
|
||||
fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks
|
||||
# of size 512 free, so succeed
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
|
||||
libc.fallocate(88, 1023)
|
||||
|
||||
def test_not_enough_space_small_blocks(self, sys_fallocate_mock,
|
||||
fstatvfs_mock):
|
||||
# Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks
|
||||
# of size 512 free, so fail
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1024')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
libc.fallocate(88, 1024)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 2048 bytes in reserve but have only 3 blocks of size 512, so
|
||||
# allocating even 0 bytes fails
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('2048')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(512, 3)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
libc.fallocate(88, 0)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Filesystem is empty, but our reserve is bigger than the
|
||||
# filesystem, so any allocation will fail
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('9999999999999')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
|
||||
self.assertRaises(OSError, libc.fallocate, 88, 0)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
|
||||
# and file size is 2047, so succeed
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1%')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
|
||||
libc.fallocate(88, 2047)
|
||||
|
||||
def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
|
||||
# and file size is 2048, so fail
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('1%')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
libc.fallocate(88, 2048)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 1 <= 1'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock):
|
||||
# Filesystem is empty, but our reserve is the whole filesystem, so
|
||||
# any allocation will fail
|
||||
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
|
||||
libc.config_fallocate_value('100%')
|
||||
|
||||
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
|
||||
with self.assertRaises(OSError) as catcher:
|
||||
libc.fallocate(88, 0)
|
||||
self.assertEqual(
|
||||
str(catcher.exception),
|
||||
'[Errno %d] FALLOCATE_RESERVE fail 100 <= 100'
|
||||
% errno.ENOSPC)
|
||||
sys_fallocate_mock.assert_not_called()
|
||||
|
||||
|
||||
@mock.patch('ctypes.get_errno')
|
||||
@mock.patch.object(libc, '_sys_fallocate')
|
||||
class TestPunchHole(unittest.TestCase):
|
||||
def test_punch_hole(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
libc.punch_hole(123, 456, 789)
|
||||
|
||||
calls = sys_fallocate_mock.mock_calls
|
||||
self.assertEqual(len(calls), 1)
|
||||
args = calls[0][1]
|
||||
self.assertEqual(len(args), 4)
|
||||
self.assertEqual(args[0], 123)
|
||||
self.assertEqual(
|
||||
args[1], libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE)
|
||||
self.assertEqual(args[2].value, 456)
|
||||
self.assertEqual(args[3].value, 789)
|
||||
|
||||
def test_error(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = -1
|
||||
get_errno_mock.return_value = errno.EISDIR
|
||||
|
||||
with self.assertRaises(OSError) as cm:
|
||||
libc.punch_hole(123, 456, 789)
|
||||
self.assertEqual(cm.exception.errno, errno.EISDIR)
|
||||
|
||||
def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = True
|
||||
sys_fallocate_mock.return_value = 0
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
libc.punch_hole(0, 1, -1)
|
||||
with self.assertRaises(ValueError):
|
||||
libc.punch_hole(0, 1 << 64, 1)
|
||||
with self.assertRaises(ValueError):
|
||||
libc.punch_hole(0, -1, 1)
|
||||
with self.assertRaises(ValueError):
|
||||
libc.punch_hole(0, 1, 0)
|
||||
with self.assertRaises(ValueError):
|
||||
libc.punch_hole(0, 1, 1 << 64)
|
||||
self.assertEqual([], sys_fallocate_mock.mock_calls)
|
||||
|
||||
# sanity check
|
||||
libc.punch_hole(0, 0, 1)
|
||||
self.assertEqual(
|
||||
[mock.call(
|
||||
0, libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE,
|
||||
mock.ANY, mock.ANY)],
|
||||
sys_fallocate_mock.mock_calls)
|
||||
# Go confirm the ctypes values separately; apparently == doesn't
|
||||
# work the way you'd expect with ctypes :-/
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
|
||||
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1)
|
||||
|
||||
def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock):
|
||||
sys_fallocate_mock.available = False
|
||||
|
||||
with self.assertRaises(OSError) as cm:
|
||||
libc.punch_hole(123, 456, 789)
|
||||
self.assertEqual(cm.exception.errno, errno.ENOTSUP)
|
||||
|
||||
|
||||
class TestPunchHoleReally(unittest.TestCase):
|
||||
def setUp(self):
|
||||
if not libc._sys_fallocate.available:
|
||||
raise unittest.SkipTest("libc._sys_fallocate not available")
|
||||
|
||||
def test_punch_a_hole(self):
|
||||
with tempfile.TemporaryFile() as tf:
|
||||
tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64)
|
||||
tf.flush()
|
||||
|
||||
# knock out the first half of the "y"s
|
||||
libc.punch_hole(tf.fileno(), 64, 32)
|
||||
|
||||
tf.seek(0)
|
||||
contents = tf.read(4096)
|
||||
self.assertEqual(
|
||||
contents,
|
||||
b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64)
|
||||
|
||||
|
||||
class Test_LibcWrapper(unittest.TestCase):
|
||||
def test_available_function(self):
|
||||
# This should pretty much always exist
|
||||
|
||||
@ -50,7 +50,6 @@ from test.unit import (mock as unit_mock, temptree, mock_check_drive,
|
||||
encode_frag_archive_bodies, skip_if_no_xattrs)
|
||||
from swift.obj import diskfile
|
||||
from swift.common import utils
|
||||
from swift.common.utils import libc
|
||||
from swift.common.utils import hash_path, mkdirs, Timestamp, \
|
||||
encode_timestamps, O_TMPFILE, md5 as _md5
|
||||
from swift.common import ring
|
||||
@ -4989,7 +4988,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
|
||||
# This is a horrible hack so you can run this test in isolation.
|
||||
# Some of the ctypes machinery calls os.close(), and that runs afoul
|
||||
# of our mock.
|
||||
with mock.patch.object(libc, '_sys_fallocate', None):
|
||||
with mock.patch.object(utils, '_sys_fallocate', None):
|
||||
utils.disable_fallocate()
|
||||
|
||||
df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user