openstack/swift
Code Issues Proposed changes

Allows to configure the rsync modules where the replicators will send data

Browse Source 
Currently, the rsync module where the replicators send data is static. It
forbids administrators to set rsync configuration based on their current
deployment or needs.

As an example, the rsyncd configuration example encourages to set a connections
limit for the modules account, container and object. It permits to protect
devices from excessives parallels connections, because it would impact
performances.

On a server with many devices, it is tempting to increase this number
proportionally, but nothing guarantees that the distribution of the connections
will be balanced. In the worst scenario, a single device can receive all the
connections, which is a severe impact on performances.

This commit adds a new option named 'rsync_module' to the *-replicator sections
of the *-server configuration file. This configuration variable can be
extrapolated with device attributes like ip, port, device, zone, ... by using
the format {NAME}. eg:
    rsync_module = {replication_ip}::object_{device}

With this configuration, an administrators can solve the problem of connections
distribution by creating one module per device in rsyncd configuration.

The default values are backward compatible:
    {replication_ip}::account
    {replication_ip}::container
    {replication_ip}::object

Option vm_test_mode is deprecated by this commit, but backward compatibility is
maintained. The option is only effective when rsync_module is not set. In that
case, {replication_port} is appended to the default value of rsync_module.

Change-Id: Iad91df50dadbe96c921181797799b4444323ce2e
This commit is contained in:
Romain LE DISEZ 2015-06-16 12:47:26 +02:00
parent bddfc521d8
commit 71f6fd025e
27 changed files with 247 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/manpages/account-server.conf.5
View File

@ -176,8 +176,6 @@ Syslog log facility. The default is LOG_LOCAL0.
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBvm_test_mode\fR
Indicates that you are using a VM environment. The default is no.
.IP \fBper_diff\fR
The default is 1000.
.IP \fBmax_diffs\fR

2
doc/manpages/container-server.conf.5
View File

@ -182,8 +182,6 @@ Syslog log facility. The default is LOG_LOCAL0.
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBvm_test_mode\fR
Indicates that you are using a VM environment. The default is no.
.IP \fBer_diff\fR
The default is 1000.
.IP \fBmax_diffs\fR

2
doc/manpages/object-server.conf.5
View File

@ -185,8 +185,6 @@ Syslog log facility. The default is LOG_LOCAL0.
Logging level. The default is INFO.
.IP \fBlog_address\fR
Logging address. The default is /dev/log.
.IP \fBvm_test_mode\fR
Indicates that you are using a VM environment. The default is no.
.IP \fBdaemonize\fR
Whether or not to run replication as a daemon. The default is yes.
.IP "\fBrun_pause [deprecated]\fR"

2
doc/saio/swift/account-server/1.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#account
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
[account-auditor]

2
doc/saio/swift/account-server/2.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#account
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
[account-auditor]

2
doc/saio/swift/account-server/3.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#account
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
[account-auditor]

2
doc/saio/swift/account-server/4.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#account
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
[account-auditor]

2
doc/saio/swift/container-server/1.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#container
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
[container-updater]

2
doc/saio/swift/container-server/2.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#container
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
[container-updater]

2
doc/saio/swift/container-server/3.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#container
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
[container-updater]

2
doc/saio/swift/container-server/4.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#container
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
[container-updater]

2
doc/saio/swift/object-server/1.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#object
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
[object-reconstructor]

2
doc/saio/swift/object-server/2.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#object
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
[object-reconstructor]

2
doc/saio/swift/object-server/3.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#object
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
[object-reconstructor]

2
doc/saio/swift/object-server/4.conf
View File

@ -20,7 +20,7 @@ use = egg:swift#object
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
[object-reconstructor]

35
doc/source/deployment_guide.rst
View File

@ -594,6 +594,17 @@ node_timeout DEFAULT or 10 Request timeout to external services.
in the DEFAULT section, or 10 (though
other sections use 3 as the final
default).
rsync_module {replication_ip}::object
Format of the rsync module where the
replicator will send data. The
configuration value can include some
variables that will be extracted from
the ring. Variables must follow the
format {NAME} where NAME is one of:
ip, port, replication_ip,
replication_port, region, zone, device,
meta. See etc/rsyncd.conf-sample for
some examples.
================== ================= =======================================
[object-updater]
@ -723,6 +734,18 @@ conn_timeout 0.5 Connection timeout to external
services
reclaim_age 604800 Time elapsed in seconds before a
container can be reclaimed
rsync_module {replication_ip}::container
Format of the rsync module where the
replicator will send data. The
configuration value can include some
variables that will be extracted from
the ring. Variables must follow the
format {NAME} where NAME is one of:
ip, port, replication_ip,
replication_port, region, zone,
device, meta. See
etc/rsyncd.conf-sample for some
examples.
================== ==================== ====================================
[container-updater]
@ -850,6 +873,18 @@ node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
reclaim_age 604800 Time elapsed in seconds before an
account can be reclaimed
rsync_module {replication_ip}::account
Format of the rsync module where the
replicator will send data. The
configuration value can include some
variables that will be extracted from
the ring. Variables must follow the
format {NAME} where NAME is one of:
ip, port, replication_ip,
replication_port, region, zone,
device, meta. See
etc/rsyncd.conf-sample for some
examples.
================== ================== ======================================
[account-auditor]

26
doc/source/replication_network.rst
View File

@ -146,7 +146,7 @@ For SAIO replication
delete all configuration options in section [<*>-replicator]
#. Add configuration files for object-server, in /etc/swift/objec-server/
#. Add configuration files for object-server, in /etc/swift/object-server/
* 5.conf::
@ -170,7 +170,7 @@ For SAIO replication
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
* 6.conf::
@ -194,7 +194,7 @@ For SAIO replication
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
* 7.conf::
@ -218,7 +218,7 @@ For SAIO replication
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
* 8.conf::
@ -242,7 +242,7 @@ For SAIO replication
use = egg:swift#recon
[object-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::object{replication_port}
#. Add configuration files for container-server, in /etc/swift/container-server/
@ -268,7 +268,7 @@ For SAIO replication
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
* 6.conf::
@ -292,7 +292,7 @@ For SAIO replication
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
* 7.conf::
@ -316,7 +316,7 @@ For SAIO replication
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
* 8.conf::
@ -340,7 +340,7 @@ For SAIO replication
use = egg:swift#recon
[container-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::container{replication_port}
#. Add configuration files for account-server, in /etc/swift/account-server/
@ -366,7 +366,7 @@ For SAIO replication
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
* 6.conf::
@ -390,7 +390,7 @@ For SAIO replication
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
* 7.conf::
@ -414,7 +414,7 @@ For SAIO replication
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
* 8.conf::
@ -438,7 +438,7 @@ For SAIO replication
use = egg:swift#recon
[account-replicator]
vm_test_mode = yes
rsync_module = {replication_ip}::account{replication_port}
---------------------------------

5
etc/account-server.conf-sample
View File

@ -90,7 +90,6 @@ use = egg:swift#recon
# log_level = INFO
# log_address = /dev/log
#
# vm_test_mode = no
# per_diff = 1000
# max_diffs = 100
# concurrency = 8
@ -111,6 +110,10 @@ use = egg:swift#recon
# a different region than the local one.
# rsync_compress = no
#
# Format of the rysnc module where the replicator will send data. See
# etc/rsyncd.conf-sample for some usage examples.
# rsync_module = {replication_ip}::account
#
# recon_cache_path = /var/cache/swift
[account-auditor]

5
etc/container-server.conf-sample
View File

@ -99,7 +99,6 @@ use = egg:swift#recon
# log_level = INFO
# log_address = /dev/log
#
# vm_test_mode = no
# per_diff = 1000
# max_diffs = 100
# concurrency = 8
@ -120,6 +119,10 @@ use = egg:swift#recon
# a different region than the local one.
# rsync_compress = no
#
# Format of the rysnc module where the replicator will send data. See
# etc/rsyncd.conf-sample for some usage examples.
# rsync_module = {replication_ip}::container
#
# recon_cache_path = /var/cache/swift
[container-updater]

5
etc/object-server.conf-sample
View File

@ -162,7 +162,6 @@ use = egg:swift#recon
# log_level = INFO
# log_address = /dev/log
#
# vm_test_mode = no
# daemonize = on
#
# Time in seconds to wait between replication passes
@ -195,6 +194,10 @@ use = egg:swift#recon
# slow down the syncing process.
# rsync_compress = no
#
# Format of the rysnc module where the replicator will send data. See
# etc/rsyncd.conf-sample for some usage examples.
# rsync_module = {replication_ip}::object
#
# node_timeout = <whatever's in the DEFAULT section or 10>
# max duration of an http request; this is for REPLICATE finalization calls and
# so should be longer than node_timeout

56
etc/rsyncd.conf-sample
View File

@ -20,3 +20,59 @@ max connections = 8
path = /srv/node
read only = false
lock file = /var/lock/object.lock
# If rsync_module includes the device, you can tune rsyncd to permit 4
# connections per device instead of simply allowing 8 connections for all
# devices:
# rsync_module = {replication_ip}::object_{device}
#
# (if devices in your object ring are named sda, sdb and sdc)
#
#[object_sda]
#max connections = 4
#path = /srv/node
#read only = false
#lock file = /var/lock/object_sda.lock
#
#[object_sdb]
#max connections = 4
#path = /srv/node
#read only = false
#lock file = /var/lock/object_sdb.lock
#
#[object_sdc]
#max connections = 4
#path = /srv/node
#read only = false
#lock file = /var/lock/object_sdc.lock
# To emulate the deprecated option vm_test_mode = yes, set:
# rsync_module = {replication_ip}::object{replication_port}
#
# So, on your SAIO, you have to set the following rsyncd configuration:
#
#[object6010]
#max connections = 25
#path = /srv/1/node/
#read only = false
#lock file = /var/lock/object6010.lock
#
#[object6020]
#max connections = 25
#path = /srv/2/node/
#read only = false
#lock file = /var/lock/object6020.lock
#
#[object6030]
#max connections = 25
#path = /srv/3/node/
#read only = false
#lock file = /var/lock/object6030.lock
#
#[object6040]
#max connections = 25
#path = /srv/4/node/
#read only = false
#lock file = /var/lock/object6040.lock

25
swift/common/db_replicator.py
View File

@ -31,7 +31,8 @@ import swift.common.db
from swift.common.direct_client import quote
from swift.common.utils import get_logger, whataremyips, storage_directory, \
renamer, mkdirs, lock_parent_directory, config_true_value, \
unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp
unlink_older_than, dump_recon_cache, rsync_module_interpolation, ismount, \
json, Timestamp
from swift.common import ring
from swift.common.ring.utils import is_local_device
from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
@ -165,11 +166,20 @@ class Replicator(Daemon):
self.max_diffs = int(conf.get('max_diffs') or 100)
self.interval = int(conf.get('interval') or
conf.get('run_pause') or 30)
self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.rsync_compress = config_true_value(
conf.get('rsync_compress', 'no'))
self.rsync_module = conf.get('rsync_module', '').rstrip('/')
if not self.rsync_module:
self.rsync_module = '{replication_ip}::%s' % self.server_type
if config_true_value(conf.get('vm_test_mode', 'no')):
self.logger.warn('Option %(type)s-replicator/vm_test_mode is '
'deprecated and will be removed in a future '
'version. Update your configuration to use '
'option %(type)s-replicator/rsync_module.'
% {'type': self.server_type})
self.rsync_module += '{replication_port}'
self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
@ -267,14 +277,9 @@ class Replicator(Daemon):
:param different_region: if True, the destination node is in a
different region
"""
device_ip = rsync_ip(device['replication_ip'])
if self.vm_test_mode:
remote_file = '%s::%s%s/%s/tmp/%s' % (
device_ip, self.server_type, device['replication_port'],
device['device'], local_id)
else:
remote_file = '%s::%s/%s/tmp/%s' % (
device_ip, self.server_type, device['device'], local_id)
rsync_module = rsync_module_interpolation(self.rsync_module, device)
rsync_path = '%s/tmp/%s' % (device['device'], local_id)
remote_file = '%s/%s' % (rsync_module, rsync_path)
mtime = os.path.getmtime(broker.db_file)
if not self._rsync_file(broker.db_file, remote_file,
different_region=different_region):

27
swift/common/utils.py
View File

@ -2703,6 +2703,33 @@ def rsync_ip(ip):
return '[%s]' % ip
def rsync_module_interpolation(template, device):
"""
Interpolate devices variables inside a rsync module template
:param template: rsync module template as a string
:param device: a device from a ring
:returns: a string with all variables replaced by device attributes
"""
replacements = {
'ip': rsync_ip(device.get('ip', '')),
'port': device.get('port', ''),
'replication_ip': rsync_ip(device.get('replication_ip', '')),
'replication_port': device.get('replication_port', ''),
'region': device.get('region', ''),
'zone': device.get('zone', ''),
'device': device.get('device', ''),
'meta': device.get('meta', ''),
}
try:
module = template.format(**replacements)
except KeyError as e:
raise ValueError('Cannot interpolate rsync_module, invalid variable: '
'%s' % e)
return module
def get_valid_utf8_str(str_or_unicode):
"""
Get valid parts of utf-8 str from str, unicode and even invalid utf-8 str

20
swift/obj/replicator.py
View File

@ -31,8 +31,8 @@ from eventlet.support.greenlets import GreenletExit
from swift.common.ring.utils import is_local_device
from swift.common.utils import whataremyips, unlink_older_than, \
compute_eta, get_logger, dump_recon_cache, ismount, \
rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \
tpool_reraise, config_auto_int_value, storage_directory
rsync_module_interpolation, mkdirs, config_true_value, list_from_csv, \
get_hub, tpool_reraise, config_auto_int_value, storage_directory
from swift.common.bufferedhttp import http_connect
from swift.common.daemon import Daemon
from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
@ -62,7 +62,6 @@ class ObjectReplicator(Daemon):
self.logger = logger or get_logger(conf, log_route='object-replicator')
self.devices_dir = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.bind_ip = conf.get('bind_ip', '0.0.0.0')
self.servers_per_port = int(conf.get('servers_per_port', '0') or 0)
@ -81,6 +80,15 @@ class ObjectReplicator(Daemon):
self.rsync_bwlimit = conf.get('rsync_bwlimit', '0')
self.rsync_compress = config_true_value(
conf.get('rsync_compress', 'no'))
self.rsync_module = conf.get('rsync_module', '').rstrip('/')
if not self.rsync_module:
self.rsync_module = '{replication_ip}::object'
if config_true_value(conf.get('vm_test_mode', 'no')):
self.logger.warn('Option object-replicator/vm_test_mode is '
'deprecated and will be removed in a future '
'version. Update your configuration to use '
'option object-replicator/rsync_module.')
self.rsync_module += '{replication_port}'
self.http_timeout = int(conf.get('http_timeout', 60))
self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
self.recon_cache_path = conf.get('recon_cache_path',
@ -223,11 +231,7 @@ class ObjectReplicator(Daemon):
# Allow for compression, but only if the remote node is in
# a different region than the local one.
args.append('--compress')
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
rsync_module = rsync_module_interpolation(self.rsync_module, node)
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)

14
test/probe/common.py
View File

@ -29,7 +29,8 @@ from six.moves.http_client import HTTPConnection
from swiftclient import get_auth, head_account
from swift.obj.diskfile import get_data_dir
from swift.common.ring import Ring
from swift.common.utils import readconf, renamer
from swift.common.utils import readconf, renamer, \
config_true_value, rsync_module_interpolation
from swift.common.manager import Manager
from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
@ -219,11 +220,12 @@ def get_ring(ring_name, required_replicas, required_devices,
"unable to find ring device %s under %s's devices (%s)" % (
dev['device'], server, conf['devices']))
# verify server is exposing rsync device
if conf.get('vm_test_mode', False):
rsync_export = '%s%s' % (server, dev['replication_port'])
else:
rsync_export = server
cmd = "rsync rsync://localhost/%s" % rsync_export
rsync_export = conf.get('rsync_module', '').rstrip('/')
if not rsync_export:
rsync_export = '{replication_ip}::%s' % server
if config_true_value(conf.get('vm_test_mode', 'no')):
rsync_export += '{replication_port}'
cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev)
p = Popen(cmd, shell=True, stdout=PIPE)
stdout, _stderr = p.communicate()
if p.returncode:

15
test/unit/common/test_db_replicator.py
View File

@ -364,10 +364,6 @@ class TestDBReplicator(unittest.TestCase):
'replication_ip': '127.0.0.1', 'replication_port': '0',
'device': 'sda1'}
def mock_rsync_ip(ip):
self.assertEquals(fake_device['ip'], ip)
return 'rsync_ip(%s)' % ip
class MyTestReplicator(TestReplicator):
def __init__(self, db_file, remote_file):
super(MyTestReplicator, self).__init__({})
@ -381,21 +377,12 @@ class TestDBReplicator(unittest.TestCase):
self_._rsync_file_called = True
return False
with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip):
broker = FakeBroker()
remote_file = 'rsync_ip(127.0.0.1)::container/sda1/tmp/abcd'
remote_file = '127.0.0.1::container/sda1/tmp/abcd'
replicator = MyTestReplicator(broker.db_file, remote_file)
replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd')
self.assertTrue(replicator._rsync_file_called)
with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip):
broker = FakeBroker()
remote_file = 'rsync_ip(127.0.0.1)::container0/sda1/tmp/abcd'
replicator = MyTestReplicator(broker.db_file, remote_file)
replicator.vm_test_mode = True
replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd')
self.assertTrue(replicator._rsync_file_called)
def test_rsync_db_rsync_file_failure(self):
class MyTestReplicator(TestReplicator):
def __init__(self):

52
test/unit/common/test_utils.py
View File

@ -2244,6 +2244,58 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertEqual(
utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]')
def test_rsync_module_interpolation(self):
fake_device = {'ip': '127.0.0.1', 'port': 11,
'replication_ip': '127.0.0.2', 'replication_port': 12,
'region': '1', 'zone': '2', 'device': 'sda1',
'meta': 'just_a_string'}
self.assertEqual(
utils.rsync_module_interpolation('{ip}', fake_device),
'127.0.0.1')
self.assertEqual(
utils.rsync_module_interpolation('{port}', fake_device),
'11')
self.assertEqual(
utils.rsync_module_interpolation('{replication_ip}', fake_device),
'127.0.0.2')
self.assertEqual(
utils.rsync_module_interpolation('{replication_port}',
fake_device),
'12')
self.assertEqual(
utils.rsync_module_interpolation('{region}', fake_device),
'1')
self.assertEqual(
utils.rsync_module_interpolation('{zone}', fake_device),
'2')
self.assertEqual(
utils.rsync_module_interpolation('{device}', fake_device),
'sda1')
self.assertEqual(
utils.rsync_module_interpolation('{meta}', fake_device),
'just_a_string')
self.assertEqual(
utils.rsync_module_interpolation('{replication_ip}::object',
fake_device),
'127.0.0.2::object')
self.assertEqual(
utils.rsync_module_interpolation('{ip}::container{port}',
fake_device),
'127.0.0.1::container11')
self.assertEqual(
utils.rsync_module_interpolation(
'{replication_ip}::object_{device}', fake_device),
'127.0.0.2::object_sda1')
self.assertEqual(
utils.rsync_module_interpolation(
'127.0.0.3::object_{replication_port}', fake_device),
'127.0.0.3::object_12')
self.assertRaises(ValueError, utils.rsync_module_interpolation,
'{replication_ip}::object_{deivce}', fake_device)
def test_fallocate_reserve(self):
class StatVFS(object):