From 71f6fd025ea013bc457450a5cc4f8e92c6df92cc Mon Sep 17 00:00:00 2001 From: Romain LE DISEZ Date: Tue, 16 Jun 2015 12:47:26 +0200 Subject: [PATCH] Allows to configure the rsync modules where the replicators will send data Currently, the rsync module where the replicators send data is static. It forbids administrators to set rsync configuration based on their current deployment or needs. As an example, the rsyncd configuration example encourages to set a connections limit for the modules account, container and object. It permits to protect devices from excessives parallels connections, because it would impact performances. On a server with many devices, it is tempting to increase this number proportionally, but nothing guarantees that the distribution of the connections will be balanced. In the worst scenario, a single device can receive all the connections, which is a severe impact on performances. This commit adds a new option named 'rsync_module' to the *-replicator sections of the *-server configuration file. This configuration variable can be extrapolated with device attributes like ip, port, device, zone, ... by using the format {NAME}. eg: rsync_module = {replication_ip}::object_{device} With this configuration, an administrators can solve the problem of connections distribution by creating one module per device in rsyncd configuration. The default values are backward compatible: {replication_ip}::account {replication_ip}::container {replication_ip}::object Option vm_test_mode is deprecated by this commit, but backward compatibility is maintained. The option is only effective when rsync_module is not set. In that case, {replication_port} is appended to the default value of rsync_module. Change-Id: Iad91df50dadbe96c921181797799b4444323ce2e --- doc/manpages/account-server.conf.5 | 2 - doc/manpages/container-server.conf.5 | 2 - doc/manpages/object-server.conf.5 | 2 - doc/saio/swift/account-server/1.conf | 2 +- doc/saio/swift/account-server/2.conf | 2 +- doc/saio/swift/account-server/3.conf | 2 +- doc/saio/swift/account-server/4.conf | 2 +- doc/saio/swift/container-server/1.conf | 2 +- doc/saio/swift/container-server/2.conf | 2 +- doc/saio/swift/container-server/3.conf | 2 +- doc/saio/swift/container-server/4.conf | 2 +- doc/saio/swift/object-server/1.conf | 2 +- doc/saio/swift/object-server/2.conf | 2 +- doc/saio/swift/object-server/3.conf | 2 +- doc/saio/swift/object-server/4.conf | 2 +- doc/source/deployment_guide.rst | 35 ++++++++++++++++ doc/source/replication_network.rst | 26 ++++++------ etc/account-server.conf-sample | 5 ++- etc/container-server.conf-sample | 5 ++- etc/object-server.conf-sample | 5 ++- etc/rsyncd.conf-sample | 56 ++++++++++++++++++++++++++ swift/common/db_replicator.py | 25 +++++++----- swift/common/utils.py | 27 +++++++++++++ swift/obj/replicator.py | 20 +++++---- test/probe/common.py | 14 ++++--- test/unit/common/test_db_replicator.py | 23 +++-------- test/unit/common/test_utils.py | 52 ++++++++++++++++++++++++ 27 files changed, 247 insertions(+), 76 deletions(-) diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index cec0752999..955eedfa76 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -176,8 +176,6 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. -.IP \fBvm_test_mode\fR -Indicates that you are using a VM environment. The default is no. .IP \fBper_diff\fR The default is 1000. .IP \fBmax_diffs\fR diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 2cd1623dc1..65981308bd 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -182,8 +182,6 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. -.IP \fBvm_test_mode\fR -Indicates that you are using a VM environment. The default is no. .IP \fBer_diff\fR The default is 1000. .IP \fBmax_diffs\fR diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5 index e82a56ffb5..c790729c16 100644 --- a/doc/manpages/object-server.conf.5 +++ b/doc/manpages/object-server.conf.5 @@ -185,8 +185,6 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. -.IP \fBvm_test_mode\fR -Indicates that you are using a VM environment. The default is no. .IP \fBdaemonize\fR Whether or not to run replication as a daemon. The default is yes. .IP "\fBrun_pause [deprecated]\fR" diff --git a/doc/saio/swift/account-server/1.conf b/doc/saio/swift/account-server/1.conf index ce599a7c8d..91ded8905d 100644 --- a/doc/saio/swift/account-server/1.conf +++ b/doc/saio/swift/account-server/1.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/account-server/2.conf b/doc/saio/swift/account-server/2.conf index 0593e9ad39..70f8fae896 100644 --- a/doc/saio/swift/account-server/2.conf +++ b/doc/saio/swift/account-server/2.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/account-server/3.conf b/doc/saio/swift/account-server/3.conf index abc27c2580..6090883bd7 100644 --- a/doc/saio/swift/account-server/3.conf +++ b/doc/saio/swift/account-server/3.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/account-server/4.conf b/doc/saio/swift/account-server/4.conf index cda9ac87b9..e435842d1a 100644 --- a/doc/saio/swift/account-server/4.conf +++ b/doc/saio/swift/account-server/4.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/container-server/1.conf b/doc/saio/swift/container-server/1.conf index 176096dbe1..5bf3c0f28c 100644 --- a/doc/saio/swift/container-server/1.conf +++ b/doc/saio/swift/container-server/1.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/container-server/2.conf b/doc/saio/swift/container-server/2.conf index 7100710b3c..7046236fae 100644 --- a/doc/saio/swift/container-server/2.conf +++ b/doc/saio/swift/container-server/2.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/container-server/3.conf b/doc/saio/swift/container-server/3.conf index 06ec47414d..c7f726199c 100644 --- a/doc/saio/swift/container-server/3.conf +++ b/doc/saio/swift/container-server/3.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/container-server/4.conf b/doc/saio/swift/container-server/4.conf index 1acc3b5c54..ff9c0e806a 100644 --- a/doc/saio/swift/container-server/4.conf +++ b/doc/saio/swift/container-server/4.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/object-server/1.conf b/doc/saio/swift/object-server/1.conf index 178e3fcba3..c6d1b1464f 100644 --- a/doc/saio/swift/object-server/1.conf +++ b/doc/saio/swift/object-server/1.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/saio/swift/object-server/2.conf b/doc/saio/swift/object-server/2.conf index 6b611ca25a..3d863cf59b 100644 --- a/doc/saio/swift/object-server/2.conf +++ b/doc/saio/swift/object-server/2.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/saio/swift/object-server/3.conf b/doc/saio/swift/object-server/3.conf index 7352592319..d99995bb5e 100644 --- a/doc/saio/swift/object-server/3.conf +++ b/doc/saio/swift/object-server/3.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/saio/swift/object-server/4.conf b/doc/saio/swift/object-server/4.conf index be1211047b..9f6fd1fc13 100644 --- a/doc/saio/swift/object-server/4.conf +++ b/doc/saio/swift/object-server/4.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index da9d0b4fa2..1516d083ca 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -594,6 +594,17 @@ node_timeout DEFAULT or 10 Request timeout to external services. in the DEFAULT section, or 10 (though other sections use 3 as the final default). +rsync_module {replication_ip}::object + Format of the rsync module where the + replicator will send data. The + configuration value can include some + variables that will be extracted from + the ring. Variables must follow the + format {NAME} where NAME is one of: + ip, port, replication_ip, + replication_port, region, zone, device, + meta. See etc/rsyncd.conf-sample for + some examples. ================== ================= ======================================= [object-updater] @@ -723,6 +734,18 @@ conn_timeout 0.5 Connection timeout to external services reclaim_age 604800 Time elapsed in seconds before a container can be reclaimed +rsync_module {replication_ip}::container + Format of the rsync module where the + replicator will send data. The + configuration value can include some + variables that will be extracted from + the ring. Variables must follow the + format {NAME} where NAME is one of: + ip, port, replication_ip, + replication_port, region, zone, + device, meta. See + etc/rsyncd.conf-sample for some + examples. ================== ==================== ==================================== [container-updater] @@ -850,6 +873,18 @@ node_timeout 10 Request timeout to external services conn_timeout 0.5 Connection timeout to external services reclaim_age 604800 Time elapsed in seconds before an account can be reclaimed +rsync_module {replication_ip}::account + Format of the rsync module where the + replicator will send data. The + configuration value can include some + variables that will be extracted from + the ring. Variables must follow the + format {NAME} where NAME is one of: + ip, port, replication_ip, + replication_port, region, zone, + device, meta. See + etc/rsyncd.conf-sample for some + examples. ================== ================== ====================================== [account-auditor] diff --git a/doc/source/replication_network.rst b/doc/source/replication_network.rst index 138cb42186..609ebc9688 100644 --- a/doc/source/replication_network.rst +++ b/doc/source/replication_network.rst @@ -146,7 +146,7 @@ For SAIO replication delete all configuration options in section [<*>-replicator] -#. Add configuration files for object-server, in /etc/swift/objec-server/ +#. Add configuration files for object-server, in /etc/swift/object-server/ * 5.conf:: @@ -170,7 +170,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} * 6.conf:: @@ -194,7 +194,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} * 7.conf:: @@ -218,7 +218,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} * 8.conf:: @@ -242,7 +242,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} #. Add configuration files for container-server, in /etc/swift/container-server/ @@ -268,7 +268,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} * 6.conf:: @@ -292,7 +292,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} * 7.conf:: @@ -316,7 +316,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} * 8.conf:: @@ -340,7 +340,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} #. Add configuration files for account-server, in /etc/swift/account-server/ @@ -366,7 +366,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} * 6.conf:: @@ -390,7 +390,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} * 7.conf:: @@ -414,7 +414,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} * 8.conf:: @@ -438,7 +438,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} --------------------------------- diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 8cfbfa0098..9eb61a9d96 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -90,7 +90,6 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # -# vm_test_mode = no # per_diff = 1000 # max_diffs = 100 # concurrency = 8 @@ -111,6 +110,10 @@ use = egg:swift#recon # a different region than the local one. # rsync_compress = no # +# Format of the rysnc module where the replicator will send data. See +# etc/rsyncd.conf-sample for some usage examples. +# rsync_module = {replication_ip}::account +# # recon_cache_path = /var/cache/swift [account-auditor] diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 54daee1e8e..0eebf366e0 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -99,7 +99,6 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # -# vm_test_mode = no # per_diff = 1000 # max_diffs = 100 # concurrency = 8 @@ -120,6 +119,10 @@ use = egg:swift#recon # a different region than the local one. # rsync_compress = no # +# Format of the rysnc module where the replicator will send data. See +# etc/rsyncd.conf-sample for some usage examples. +# rsync_module = {replication_ip}::container +# # recon_cache_path = /var/cache/swift [container-updater] diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 31bd160a3e..84328b0c3f 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -162,7 +162,6 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # -# vm_test_mode = no # daemonize = on # # Time in seconds to wait between replication passes @@ -195,6 +194,10 @@ use = egg:swift#recon # slow down the syncing process. # rsync_compress = no # +# Format of the rysnc module where the replicator will send data. See +# etc/rsyncd.conf-sample for some usage examples. +# rsync_module = {replication_ip}::object +# # node_timeout = # max duration of an http request; this is for REPLICATE finalization calls and # so should be longer than node_timeout diff --git a/etc/rsyncd.conf-sample b/etc/rsyncd.conf-sample index c3b9952b16..103a85a57b 100644 --- a/etc/rsyncd.conf-sample +++ b/etc/rsyncd.conf-sample @@ -20,3 +20,59 @@ max connections = 8 path = /srv/node read only = false lock file = /var/lock/object.lock + + +# If rsync_module includes the device, you can tune rsyncd to permit 4 +# connections per device instead of simply allowing 8 connections for all +# devices: +# rsync_module = {replication_ip}::object_{device} +# +# (if devices in your object ring are named sda, sdb and sdc) +# +#[object_sda] +#max connections = 4 +#path = /srv/node +#read only = false +#lock file = /var/lock/object_sda.lock +# +#[object_sdb] +#max connections = 4 +#path = /srv/node +#read only = false +#lock file = /var/lock/object_sdb.lock +# +#[object_sdc] +#max connections = 4 +#path = /srv/node +#read only = false +#lock file = /var/lock/object_sdc.lock + + +# To emulate the deprecated option vm_test_mode = yes, set: +# rsync_module = {replication_ip}::object{replication_port} +# +# So, on your SAIO, you have to set the following rsyncd configuration: +# +#[object6010] +#max connections = 25 +#path = /srv/1/node/ +#read only = false +#lock file = /var/lock/object6010.lock +# +#[object6020] +#max connections = 25 +#path = /srv/2/node/ +#read only = false +#lock file = /var/lock/object6020.lock +# +#[object6030] +#max connections = 25 +#path = /srv/3/node/ +#read only = false +#lock file = /var/lock/object6030.lock +# +#[object6040] +#max connections = 25 +#path = /srv/4/node/ +#read only = false +#lock file = /var/lock/object6040.lock diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 7a6e8d549f..71d1634c42 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -31,7 +31,8 @@ import swift.common.db from swift.common.direct_client import quote from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ - unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp + unlink_older_than, dump_recon_cache, rsync_module_interpolation, ismount, \ + json, Timestamp from swift.common import ring from swift.common.ring.utils import is_local_device from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE @@ -165,11 +166,20 @@ class Replicator(Daemon): self.max_diffs = int(conf.get('max_diffs') or 100) self.interval = int(conf.get('interval') or conf.get('run_pause') or 30) - self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.rsync_compress = config_true_value( conf.get('rsync_compress', 'no')) + self.rsync_module = conf.get('rsync_module', '').rstrip('/') + if not self.rsync_module: + self.rsync_module = '{replication_ip}::%s' % self.server_type + if config_true_value(conf.get('vm_test_mode', 'no')): + self.logger.warn('Option %(type)s-replicator/vm_test_mode is ' + 'deprecated and will be removed in a future ' + 'version. Update your configuration to use ' + 'option %(type)s-replicator/rsync_module.' + % {'type': self.server_type}) + self.rsync_module += '{replication_port}' self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) @@ -267,14 +277,9 @@ class Replicator(Daemon): :param different_region: if True, the destination node is in a different region """ - device_ip = rsync_ip(device['replication_ip']) - if self.vm_test_mode: - remote_file = '%s::%s%s/%s/tmp/%s' % ( - device_ip, self.server_type, device['replication_port'], - device['device'], local_id) - else: - remote_file = '%s::%s/%s/tmp/%s' % ( - device_ip, self.server_type, device['device'], local_id) + rsync_module = rsync_module_interpolation(self.rsync_module, device) + rsync_path = '%s/tmp/%s' % (device['device'], local_id) + remote_file = '%s/%s' % (rsync_module, rsync_path) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file, different_region=different_region): diff --git a/swift/common/utils.py b/swift/common/utils.py index 0dbbda6c09..996486916d 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -2703,6 +2703,33 @@ def rsync_ip(ip): return '[%s]' % ip +def rsync_module_interpolation(template, device): + """ + Interpolate devices variables inside a rsync module template + + :param template: rsync module template as a string + :param device: a device from a ring + + :returns: a string with all variables replaced by device attributes + """ + replacements = { + 'ip': rsync_ip(device.get('ip', '')), + 'port': device.get('port', ''), + 'replication_ip': rsync_ip(device.get('replication_ip', '')), + 'replication_port': device.get('replication_port', ''), + 'region': device.get('region', ''), + 'zone': device.get('zone', ''), + 'device': device.get('device', ''), + 'meta': device.get('meta', ''), + } + try: + module = template.format(**replacements) + except KeyError as e: + raise ValueError('Cannot interpolate rsync_module, invalid variable: ' + '%s' % e) + return module + + def get_valid_utf8_str(str_or_unicode): """ Get valid parts of utf-8 str from str, unicode and even invalid utf-8 str diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 842a2a859c..124df3e786 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -31,8 +31,8 @@ from eventlet.support.greenlets import GreenletExit from swift.common.ring.utils import is_local_device from swift.common.utils import whataremyips, unlink_older_than, \ compute_eta, get_logger, dump_recon_cache, ismount, \ - rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \ - tpool_reraise, config_auto_int_value, storage_directory + rsync_module_interpolation, mkdirs, config_true_value, list_from_csv, \ + get_hub, tpool_reraise, config_auto_int_value, storage_directory from swift.common.bufferedhttp import http_connect from swift.common.daemon import Daemon from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE @@ -62,7 +62,6 @@ class ObjectReplicator(Daemon): self.logger = logger or get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) - self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.bind_ip = conf.get('bind_ip', '0.0.0.0') self.servers_per_port = int(conf.get('servers_per_port', '0') or 0) @@ -81,6 +80,15 @@ class ObjectReplicator(Daemon): self.rsync_bwlimit = conf.get('rsync_bwlimit', '0') self.rsync_compress = config_true_value( conf.get('rsync_compress', 'no')) + self.rsync_module = conf.get('rsync_module', '').rstrip('/') + if not self.rsync_module: + self.rsync_module = '{replication_ip}::object' + if config_true_value(conf.get('vm_test_mode', 'no')): + self.logger.warn('Option object-replicator/vm_test_mode is ' + 'deprecated and will be removed in a future ' + 'version. Update your configuration to use ' + 'option object-replicator/rsync_module.') + self.rsync_module += '{replication_port}' self.http_timeout = int(conf.get('http_timeout', 60)) self.lockup_timeout = int(conf.get('lockup_timeout', 1800)) self.recon_cache_path = conf.get('recon_cache_path', @@ -223,11 +231,7 @@ class ObjectReplicator(Daemon): # Allow for compression, but only if the remote node is in # a different region than the local one. args.append('--compress') - node_ip = rsync_ip(node['replication_ip']) - if self.vm_test_mode: - rsync_module = '%s::object%s' % (node_ip, node['replication_port']) - else: - rsync_module = '%s::object' % node_ip + rsync_module = rsync_module_interpolation(self.rsync_module, node) had_any = False for suffix in suffixes: spath = join(job['path'], suffix) diff --git a/test/probe/common.py b/test/probe/common.py index 45a907444d..37bf885536 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -29,7 +29,8 @@ from six.moves.http_client import HTTPConnection from swiftclient import get_auth, head_account from swift.obj.diskfile import get_data_dir from swift.common.ring import Ring -from swift.common.utils import readconf, renamer +from swift.common.utils import readconf, renamer, \ + config_true_value, rsync_module_interpolation from swift.common.manager import Manager from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY @@ -219,11 +220,12 @@ def get_ring(ring_name, required_replicas, required_devices, "unable to find ring device %s under %s's devices (%s)" % ( dev['device'], server, conf['devices'])) # verify server is exposing rsync device - if conf.get('vm_test_mode', False): - rsync_export = '%s%s' % (server, dev['replication_port']) - else: - rsync_export = server - cmd = "rsync rsync://localhost/%s" % rsync_export + rsync_export = conf.get('rsync_module', '').rstrip('/') + if not rsync_export: + rsync_export = '{replication_ip}::%s' % server + if config_true_value(conf.get('vm_test_mode', 'no')): + rsync_export += '{replication_port}' + cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev) p = Popen(cmd, shell=True, stdout=PIPE) stdout, _stderr = p.communicate() if p.returncode: diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 91a5adfbfd..03c3864350 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -364,10 +364,6 @@ class TestDBReplicator(unittest.TestCase): 'replication_ip': '127.0.0.1', 'replication_port': '0', 'device': 'sda1'} - def mock_rsync_ip(ip): - self.assertEquals(fake_device['ip'], ip) - return 'rsync_ip(%s)' % ip - class MyTestReplicator(TestReplicator): def __init__(self, db_file, remote_file): super(MyTestReplicator, self).__init__({}) @@ -381,20 +377,11 @@ class TestDBReplicator(unittest.TestCase): self_._rsync_file_called = True return False - with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip): - broker = FakeBroker() - remote_file = 'rsync_ip(127.0.0.1)::container/sda1/tmp/abcd' - replicator = MyTestReplicator(broker.db_file, remote_file) - replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') - self.assertTrue(replicator._rsync_file_called) - - with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip): - broker = FakeBroker() - remote_file = 'rsync_ip(127.0.0.1)::container0/sda1/tmp/abcd' - replicator = MyTestReplicator(broker.db_file, remote_file) - replicator.vm_test_mode = True - replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') - self.assertTrue(replicator._rsync_file_called) + broker = FakeBroker() + remote_file = '127.0.0.1::container/sda1/tmp/abcd' + replicator = MyTestReplicator(broker.db_file, remote_file) + replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') + self.assertTrue(replicator._rsync_file_called) def test_rsync_db_rsync_file_failure(self): class MyTestReplicator(TestReplicator): diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 653d939cce..d1fdb1f411 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2244,6 +2244,58 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEqual( utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]') + def test_rsync_module_interpolation(self): + fake_device = {'ip': '127.0.0.1', 'port': 11, + 'replication_ip': '127.0.0.2', 'replication_port': 12, + 'region': '1', 'zone': '2', 'device': 'sda1', + 'meta': 'just_a_string'} + + self.assertEqual( + utils.rsync_module_interpolation('{ip}', fake_device), + '127.0.0.1') + self.assertEqual( + utils.rsync_module_interpolation('{port}', fake_device), + '11') + self.assertEqual( + utils.rsync_module_interpolation('{replication_ip}', fake_device), + '127.0.0.2') + self.assertEqual( + utils.rsync_module_interpolation('{replication_port}', + fake_device), + '12') + self.assertEqual( + utils.rsync_module_interpolation('{region}', fake_device), + '1') + self.assertEqual( + utils.rsync_module_interpolation('{zone}', fake_device), + '2') + self.assertEqual( + utils.rsync_module_interpolation('{device}', fake_device), + 'sda1') + self.assertEqual( + utils.rsync_module_interpolation('{meta}', fake_device), + 'just_a_string') + + self.assertEqual( + utils.rsync_module_interpolation('{replication_ip}::object', + fake_device), + '127.0.0.2::object') + self.assertEqual( + utils.rsync_module_interpolation('{ip}::container{port}', + fake_device), + '127.0.0.1::container11') + self.assertEqual( + utils.rsync_module_interpolation( + '{replication_ip}::object_{device}', fake_device), + '127.0.0.2::object_sda1') + self.assertEqual( + utils.rsync_module_interpolation( + '127.0.0.3::object_{replication_port}', fake_device), + '127.0.0.3::object_12') + + self.assertRaises(ValueError, utils.rsync_module_interpolation, + '{replication_ip}::object_{deivce}', fake_device) + def test_fallocate_reserve(self): class StatVFS(object):